X86ISelLowering.cpp File Reference

#include "X86ISelLowering.h"
#include "MCTargetDesc/X86ShuffleDecode.h"
#include "X86.h"
#include "X86FrameLowering.h"
#include "X86InstrBuilder.h"
#include "X86IntrinsicsInfo.h"
#include "X86MachineFunctionInfo.h"
#include "X86TargetMachine.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/IntrinsicLowering.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/EHPersonalities.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <bitset>
#include <cctype>
#include <numeric>

Go to the source code of this file.

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.
namespace	llvm::X86
	Define some predicates that are used for node matching.

Macros
#define	DEBUG_TYPE   "x86-isel"
#define	NODE_NAME_CASE(NODE)   case X86ISD::NODE: return "X86ISD::" #NODE;
#define	GET_EGPR_IF_ENABLED(OPC)   (Subtarget.hasEGPR() ? OPC##_EVEX : OPC)

Enumerations
enum	BitTestKind : unsigned {   UndefBit , ConstantBit , NotConstantBit , ShiftBit ,   NotShiftBit }
enum	{ llvm::X86::MaxShuffleCombineDepth = 8 }
enum class	ShrinkMode { MULS8 , MULU8 , MULS16 , MULU16 }
	Different mul shrinking modes. More...

Functions
static bool	isLogicOp (unsigned Opcode)
static bool	isTargetShuffle (unsigned Opcode)
static bool	isTargetShuffleVariableMask (unsigned Opcode)
static bool	isX86CCSigned (unsigned X86CC)
	Return true if the condition is an signed comparison operation.
static X86::CondCode	TranslateIntegerX86CC (ISD::CondCode SetCCOpcode)
static X86::CondCode	TranslateX86CC (ISD::CondCode SetCCOpcode, const SDLoc &DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG)
	Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make.
static bool	hasFPCMov (unsigned X86CC)
	Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
static bool	useVPTERNLOG (const X86Subtarget &Subtarget, MVT VT)
static bool	isUndefOrEqual (int Val, int CmpVal)
	Val is the undef sentinel value or equal to the specified value.
static bool	isUndefOrEqual (ArrayRef< int > Mask, int CmpVal)
	Return true if every element in Mask is the undef sentinel value or equal to the specified value.
static bool	isUndefOrEqualInRange (ArrayRef< int > Mask, int CmpVal, unsigned Pos, unsigned Size)
	Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value or equal to the specified value.
static bool	isUndefOrZero (int Val)
	Val is either the undef or zero sentinel value.
static bool	isUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
	Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value.
static bool	isUndefLowerHalf (ArrayRef< int > Mask)
	Return true if the mask creates a vector whose lower half is undefined.
static bool	isUndefUpperHalf (ArrayRef< int > Mask)
	Return true if the mask creates a vector whose upper half is undefined.
static bool	isInRange (int Val, int Low, int Hi)
	Return true if Val falls within the specified range (L, H].
static bool	isAnyInRange (ArrayRef< int > Mask, int Low, int Hi)
	Return true if the value of any element in Mask falls within the specified range (L, H].
static bool	isAnyZero (ArrayRef< int > Mask)
	Return true if the value of any element in Mask is the zero sentinel value.
static bool	isUndefOrInRange (int Val, int Low, int Hi)
	Return true if Val is undef or if its value falls within the specified range (L, H].
static bool	isUndefOrInRange (ArrayRef< int > Mask, int Low, int Hi)
	Return true if every element in Mask is undef or if its value falls within the specified range (L, H].
static bool	isUndefOrZeroOrInRange (int Val, int Low, int Hi)
	Return true if Val is undef, zero or if its value falls within the specified range (L, H].
static bool	isUndefOrZeroOrInRange (ArrayRef< int > Mask, int Low, int Hi)
	Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H].
static LLVM_ATTRIBUTE_UNUSED bool	isBlendOrUndef (ArrayRef< int > Mask)
	Return true if every element in Mask, is an in-place blend/select mask or is undef.
static bool	isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
	Return true if every element in Mask, beginning from position Pos and ending in Pos + Size, falls within the specified sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.
static bool	isSequentialOrUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
	Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero.
static bool	isUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
	Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero.
static bool	isCompletePermute (ArrayRef< int > Mask)
	Return true if every element of a single input is referenced by the shuffle mask.
static bool	canWidenShuffleElements (ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask)
	Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.
static bool	canWidenShuffleElements (ArrayRef< int > Mask, const APInt &Zeroable, bool V2IsZero, SmallVectorImpl< int > &WidenedMask)
static bool	canWidenShuffleElements (ArrayRef< int > Mask)
static bool	scaleShuffleElements (ArrayRef< int > Mask, unsigned NumDstElts, SmallVectorImpl< int > &ScaledMask)
static bool	canScaleShuffleElements (ArrayRef< int > Mask, unsigned NumDstElts)
static SDValue	getConstVector (ArrayRef< int > Values, MVT VT, SelectionDAG &DAG, const SDLoc &dl, bool IsMask=false)
static SDValue	getConstVector (ArrayRef< APInt > Bits, const APInt &Undefs, MVT VT, SelectionDAG &DAG, const SDLoc &dl)
static SDValue	getConstVector (ArrayRef< APInt > Bits, MVT VT, SelectionDAG &DAG, const SDLoc &dl)
static SDValue	getZeroVector (MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
	Returns a vector of specified type with all zero elements.
static SDValue	getSplitVectorSrc (SDValue LHS, SDValue RHS, bool AllowCommute)
static SDValue	extractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
static SDValue	extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
	Generate a DAG to grab 128-bits from a vector > 128 bits.
static SDValue	extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
	Generate a DAG to grab 256-bits from a 512-bit vector.
static SDValue	insertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
static SDValue	insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
	Generate a DAG to put 128-bits into a vector > 128 bits.
static SDValue	widenSubVector (MVT VT, SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
	Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.
static SDValue	widenSubVector (SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl, unsigned WideSizeInBits)
	Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.
static MVT	widenMaskVectorType (MVT VT, const X86Subtarget &Subtarget)
	Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.
static SDValue	widenMaskVector (SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
	Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.
static bool	collectConcatOps (SDNode *N, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG)
static SDValue	isUpperSubvectorUndef (SDValue V, const SDLoc &DL, SelectionDAG &DAG)
static bool	isFreeToSplitVector (SDNode *N, SelectionDAG &DAG)
static std::pair< SDValue, SDValue >	splitVector (SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
static SDValue	splitVectorOp (SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
	Break an operation into 2 half sized ops and then concatenate the results.
static SDValue	splitVectorIntUnary (SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
	Break an unary integer operation into 2 half sized ops and then concatenate the result back.
static SDValue	splitVectorIntBinary (SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
	Break a binary integer operation into 2 half sized ops and then concatenate the result back.
template<typename F >
SDValue	SplitOpsAndApply (SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, F Builder, bool CheckBWI=true)
static SDValue	getAVX512Node (unsigned Opcode, const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	insert1BitVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Insert i1-subvector to i1-vector.
static SDValue	concatSubVectors (SDValue V1, SDValue V2, SelectionDAG &DAG, const SDLoc &dl)
static SDValue	getOnesVector (EVT VT, SelectionDAG &DAG, const SDLoc &dl)
	Returns a vector of specified type with all bits set.
static SDValue	getEXTEND_VECTOR_INREG (unsigned Opcode, const SDLoc &DL, EVT VT, SDValue In, SelectionDAG &DAG)
static SDValue	getBitSelect (const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS, SDValue Mask, SelectionDAG &DAG)
static SDValue	getVectorShuffle (SelectionDAG &DAG, EVT VT, const SDLoc &dl, SDValue V1, SDValue V2, ArrayRef< int > Mask)
static SDValue	getUnpackl (SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2)
	Returns a vector_shuffle node for an unpackl operation.
static SDValue	getUnpackh (SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2)
	Returns a vector_shuffle node for an unpackh operation.
static SDValue	getPack (SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS, bool PackHiHalf=false)
	Returns a node that packs the LHS + RHS nodes together at half width.
static SDValue	getShuffleVectorZeroOrUndef (SDValue V2, int Idx, bool IsZero, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Return a vector_shuffle of the specified vector of zero or undef vector.
static ConstantPoolSDNode *	getTargetConstantPoolFromBasePtr (SDValue Ptr)
static const Constant *	getTargetConstantFromBasePtr (SDValue Ptr)
static const Constant *	getTargetConstantFromNode (LoadSDNode *Load)
static const Constant *	getTargetConstantFromNode (SDValue Op)
static bool	getTargetConstantBitsFromNode (SDValue Op, unsigned EltSizeInBits, APInt &UndefElts, SmallVectorImpl< APInt > &EltBits, bool AllowWholeUndefs=true, bool AllowPartialUndefs=false)
bool	llvm::X86::isConstantSplat (SDValue Op, APInt &SplatVal, bool AllowPartialUndefs=true)
	If Op is a constant whose elements are all the same constant or undefined, return true and return the constant value in SplatVal.
static bool	getTargetShuffleMaskIndices (SDValue MaskNode, unsigned MaskEltSizeInBits, SmallVectorImpl< uint64_t > &RawMask, APInt &UndefElts)
static bool	isConstantPowerOf2 (SDValue V, unsigned EltSizeInBIts, bool AllowUndefs)
static SDValue	IsNOT (SDValue V, SelectionDAG &DAG)
static void	createPackShuffleMask (MVT VT, SmallVectorImpl< int > &Mask, bool Unary, unsigned NumStages=1)
	Create a shuffle mask that matches the PACKSS/PACKUS truncation.
static void	getPackDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
static void	getHorizDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
static bool	getTargetShuffleMask (SDValue N, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask, bool &IsUnary)
	Calculates the shuffle mask corresponding to the target-specific opcode.
static bool	getTargetShuffleMask (SDValue N, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask)
static void	computeZeroableShuffleElements (ArrayRef< int > Mask, SDValue V1, SDValue V2, APInt &KnownUndef, APInt &KnownZero)
	Compute whether each element of a shuffle is zeroable.
static bool	getTargetShuffleAndZeroables (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, APInt &KnownUndef, APInt &KnownZero)
	Decode a target shuffle mask and inputs and see if any values are known to be undef or zero from their inputs.
static void	resolveTargetShuffleFromZeroables (SmallVectorImpl< int > &Mask, const APInt &KnownUndef, const APInt &KnownZero, bool ResolveKnownZeros=true)
static void	resolveZeroablesFromTargetShuffle (const SmallVectorImpl< int > &Mask, APInt &KnownUndef, APInt &KnownZero)
static bool	createShuffleMaskFromVSELECT (SmallVectorImpl< int > &Mask, SDValue Cond, bool IsBLENDV=false)
static bool	getTargetShuffleInputs (SDValue Op, const APInt &DemandedElts, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
static bool	getFauxShuffleMask (SDValue N, const APInt &DemandedElts, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
static void	resolveTargetShuffleInputsAndMask (SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask)
	Removes unused/repeated shuffle source inputs and adjusts the shuffle mask.
static bool	getTargetShuffleInputs (SDValue Op, const APInt &DemandedElts, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, APInt &KnownUndef, APInt &KnownZero, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
	Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs and then sets the SM_SentinelUndef and SM_SentinelZero values.
static bool	getTargetShuffleInputs (SDValue Op, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, const SelectionDAG &DAG, unsigned Depth=0, bool ResolveKnownElts=true)
static SDValue	getBROADCAST_LOAD (unsigned Opcode, const SDLoc &DL, EVT VT, EVT MemVT, MemSDNode *Mem, unsigned Offset, SelectionDAG &DAG)
static SDValue	getShuffleScalarElt (SDValue Op, unsigned Index, SelectionDAG &DAG, unsigned Depth)
	Returns the scalar element that will make up the i'th element of the result of the vector shuffle.
static SDValue	LowerBuildVectorAsInsert (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerBuildVectorv16i8 (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Custom lower build_vector of v16i8.
static SDValue	LowerBuildVectorv8i16 (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Custom lower build_vector of v8i16.
static SDValue	LowerBuildVectorv4x32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Custom lower build_vector of v4i32 or v4f32.
static SDValue	getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &dl)
	Return a vector logical shift node.
static SDValue	LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, const SDLoc &dl, SelectionDAG &DAG)
static bool	findEltLoadSrc (SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset)
static SDValue	EltsFromConsecutiveLoads (EVT VT, ArrayRef< SDValue > Elts, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsAfterLegalize)
	Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.
static SDValue	combineToConsecutiveLoads (EVT VT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsAfterLegalize)
static Constant *	getConstantVector (MVT VT, ArrayRef< APInt > Bits, const APInt &Undefs, LLVMContext &C)
static Constant *	getConstantVector (MVT VT, const APInt &SplatValue, unsigned SplatBitSize, LLVMContext &C)
static bool	isFoldableUseOfShuffle (SDNode *N)
static SDValue	lowerBuildVectorAsBroadcast (BuildVectorSDNode *BVOp, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which uses: a.
static int	getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx)
	For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.
static SDValue	buildFromShuffleMostly (SDValue Op, const SDLoc &DL, SelectionDAG &DAG)
static SDValue	LowerBUILD_VECTORvXbf16 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerBUILD_VECTORvXi1 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static LLVM_ATTRIBUTE_UNUSED bool	isHorizOp (unsigned Opcode)
static bool	isHorizontalBinOpPart (const BuildVectorSDNode *N, unsigned Opcode, const SDLoc &DL, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1)
	This is a helper function of LowerToHorizontalOp().
static SDValue	ExpandHorizontalBinOp (const SDValue &V0, const SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI)
	Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.
static bool	isAddSubOrSubAdd (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, unsigned &NumExtracts, bool &IsSubAdd)
	Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB/SUBADD operation.
static bool	isFMAddSubOrFMSubAdd (const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2, unsigned ExpectedUses)
	Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBADD(Opnd0, Opnd1) into FMADDSUB/FMSUBADD(x, y, Opnd1).
static SDValue	lowerToAddSubOrFMAddSub (const BuildVectorSDNode *BV, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or X86ISD::FMSUBADD node.
static bool	isHopBuildVector (const BuildVectorSDNode *BV, SelectionDAG &DAG, unsigned &HOpcode, SDValue &V0, SDValue &V1)
static SDValue	getHopForBuildVector (const BuildVectorSDNode *BV, const SDLoc &DL, SelectionDAG &DAG, unsigned HOpcode, SDValue V0, SDValue V1)
static SDValue	LowerToHorizontalOp (const BuildVectorSDNode *BV, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
static SDValue	LowerShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerBuildVectorToBitOp (BuildVectorSDNode *Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors.
static SDValue	materializeVectorConstant (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Create a vector constant without a load.
static SDValue	createVariablePermute (MVT VT, SDValue SrcVec, SDValue IndicesVec, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source values and a vector of extraction indices.
static SDValue	LowerBUILD_VECTORAsVariablePermute (SDValue V, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerAVXCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerCONCAT_VECTORSvXi1 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerCONCAT_VECTORS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static bool	isNoopShuffleMask (ArrayRef< int > Mask)
	Tiny helper function to identify a no-op mask.
static bool	isLaneCrossingShuffleMask (unsigned LaneSizeInBits, unsigned ScalarSizeInBits, ArrayRef< int > Mask)
	Test whether there are elements crossing LaneSizeInBits lanes in this shuffle mask.
static bool	is128BitLaneCrossingShuffleMask (MVT VT, ArrayRef< int > Mask)
	Test whether there are elements crossing 128-bit lanes in this shuffle mask.
static bool	isMultiLaneShuffleMask (unsigned LaneSizeInBits, unsigned ScalarSizeInBits, ArrayRef< int > Mask)
	Test whether elements in each LaneSizeInBits lane in this shuffle mask come from multiple lanes - this is different to isLaneCrossingShuffleMask to better support 'repeated mask + lane permute' style shuffles.
static bool	isRepeatedShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
	Test whether a shuffle mask is equivalent within each sub-lane.
static bool	is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
	Test whether a shuffle mask is equivalent within each 128-bit lane.
static bool	is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask)
static bool	is256BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
	Test whether a shuffle mask is equivalent within each 256-bit lane.
static bool	isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, unsigned EltSizeInBits, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
	Test whether a target shuffle mask is equivalent within each sub-lane.
static bool	isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
	Test whether a target shuffle mask is equivalent within each sub-lane.
static bool	IsElementEquivalent (int MaskSize, SDValue Op, SDValue ExpectedOp, int Idx, int ExpectedIdx)
	Checks whether the vector elements referenced by two shuffle masks are equivalent.
static bool	isShuffleEquivalent (ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, SDValue V1=SDValue(), SDValue V2=SDValue())
	Checks whether a shuffle mask is equivalent to an explicit list of arguments.
static bool	isTargetShuffleEquivalent (MVT VT, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, const SelectionDAG &DAG, SDValue V1=SDValue(), SDValue V2=SDValue())
	Checks whether a target shuffle mask is equivalent to an explicit pattern.
static bool	isUnpackWdShuffleMask (ArrayRef< int > Mask, MVT VT, const SelectionDAG &DAG)
static bool	is128BitUnpackShuffleMask (ArrayRef< int > Mask, const SelectionDAG &DAG)
static bool	hasIdenticalHalvesShuffleMask (ArrayRef< int > Mask)
	Return true if a shuffle mask chooses elements identically in its top and bottom halves.
static unsigned	getV4X86ShuffleImm (ArrayRef< int > Mask)
	Get a 4-lane 8-bit shuffle immediate for a mask.
static SDValue	getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
static unsigned	getSHUFPDImm (ArrayRef< int > Mask)
static SDValue	getSHUFPDImmForMask (ArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
static bool	isNonZeroElementsInOrder (const APInt &Zeroable, ArrayRef< int > Mask, const EVT &VectorType, bool &IsZeroSideLeft)
static SDValue	lowerShuffleWithPSHUFB (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a shuffle with a single PSHUFB of V1 or V2.
static SDValue	getMaskNode (SDValue Mask, MVT MaskVT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
	Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics.
static SDValue	lowerShuffleWithEXPAND (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static bool	matchShuffleWithUNPCK (MVT VT, SDValue &V1, SDValue &V2, unsigned &UnpackOpcode, bool IsUnary, ArrayRef< int > TargetMask, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerShuffleWithUNPCK (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
static SDValue	lowerShuffleWithUNPCK256 (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
	Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit) followed by unpack 256-bit.
static bool	matchShuffleAsVTRUNC (MVT &SrcVT, MVT &DstVT, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget)
static SDValue	getAVX512TruncNode (const SDLoc &DL, MVT DstVT, SDValue Src, const X86Subtarget &Subtarget, SelectionDAG &DAG, bool ZeroUppers)
static SDValue	lowerShuffleWithVPMOV (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerShuffleAsVTRUNC (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static int	canLowerByDroppingElements (ArrayRef< int > Mask, bool MatchEven, bool IsSingleInput)
	Check whether a compaction lowering can be done by dropping even/odd elements and compute how many times even/odd elements must be dropped.
static bool	matchShuffleWithPACK (MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, unsigned &PackOpcode, ArrayRef< int > TargetMask, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned MaxStages=1)
static SDValue	lowerShuffleWithPACK (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerShuffleAsBitMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to emit a bitmask instruction for a shuffle.
static SDValue	lowerShuffleAsBitBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
	Try to emit a blend instruction for a shuffle using bit math.
static SDValue	getVectorMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics.
static bool	matchShuffleAsBlend (MVT VT, SDValue V1, SDValue V2, MutableArrayRef< int > Mask, const APInt &Zeroable, bool &ForceV1Zero, bool &ForceV2Zero, uint64_t &BlendMask)
static SDValue	lowerShuffleAsBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Original, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to emit a blend instruction for a shuffle.
static SDValue	lowerShuffleAsBlendAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool ImmBlends=false)
	Try to lower as a blend of elements from two inputs followed by a single-input permutation.
static SDValue	lowerShuffleAsUNPCKAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
	Try to lower as an unpack of elements from two inputs followed by a single-input permutation.
static SDValue	lowerShuffleAsPermuteAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.
static SDValue	lowerShuffleAsByteRotateAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then permuting the elements of the result in place.
static bool	isBroadcastShuffleMask (ArrayRef< int > Mask)
static bool	isNoopOrBroadcastShuffleMask (ArrayRef< int > Mask)
static bool	isSingleElementRepeatedMask (ArrayRef< int > Mask)
	Check if the Mask consists of the same element repeated multiple times.
static SDValue	lowerShuffleAsDecomposedShuffleMerge (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Generic routine to decompose a shuffle and blend into independent blends and permutes.
static int	matchShuffleAsBitRotate (MVT &RotateVT, int EltSizeInBits, const X86Subtarget &Subtarget, ArrayRef< int > Mask)
static SDValue	lowerShuffleAsBitRotate (const SDLoc &DL, MVT VT, SDValue V1, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower shuffle using X86ISD::VROTLI rotations.
static int	matchShuffleAsElementRotate (SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
	Try to match a vector shuffle as an element rotation.
static int	matchShuffleAsByteRotate (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
	Try to lower a vector shuffle as a byte rotation.
static SDValue	lowerShuffleAsByteRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerShuffleAsVALIGN (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a vector shuffle as a dword/qword rotation.
static SDValue	lowerShuffleAsByteShiftMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a vector shuffle as a byte shift sequence.
static int	matchShuffleAsShift (MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable, const X86Subtarget &Subtarget)
	Try to lower a vector shuffle as a bit shift (shifts in zeros).
static SDValue	lowerShuffleAsShift (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG, bool BitwiseOnly)
static bool	matchShuffleAsEXTRQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx, const APInt &Zeroable)
static bool	matchShuffleAsINSERTQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx)
static SDValue	lowerShuffleWithSSE4A (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
	Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
static SDValue	lowerShuffleAsSpecificZeroOrAnyExtend (const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower a vector shuffle as a zero or any extension.
static SDValue	lowerShuffleAsZeroOrAnyExtend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a vector shuffle as a zero extension on any microarch.
static SDValue	getScalarValueForVectorElement (SDValue V, int Idx, SelectionDAG &DAG)
	Try to get a scalar value for a specific element of a vector.
static bool	isShuffleFoldableLoad (SDValue V)
	Helper to test for a load that can be folded with x86 shuffles.
template<typename T >
static bool	isSoftF16 (T VT, const X86Subtarget &Subtarget)
static SDValue	lowerShuffleAsElementInsertion (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower insertion of a single element into a zero vector.
static SDValue	lowerShuffleAsTruncBroadcast (const SDLoc &DL, MVT VT, SDValue V0, int BroadcastIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements.
static bool	isSingleSHUFPSMask (ArrayRef< int > Mask)
	Test whether this can be lowered with a single SHUFPS instruction.
static bool	isShuffleMaskInputInPlace (int Input, ArrayRef< int > Mask)
	Test whether the specified input (0 or 1) is in-place blended by the given mask.
static SDValue	lowerShuffleOfExtractsAsVperm (const SDLoc &DL, SDValue N0, SDValue N1, ArrayRef< int > Mask, SelectionDAG &DAG)
	If we are extracting two 128-bit halves of a vector and shuffling the result, match that to a 256-bit AVX2 vperm* instruction to avoid a multi-shuffle lowering.
static SDValue	lowerShuffleAsBroadcast (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower broadcast of a single element.
static bool	matchShuffleAsInsertPS (SDValue &V1, SDValue &V2, unsigned &InsertPSMask, const APInt &Zeroable, ArrayRef< int > Mask, SelectionDAG &DAG)
static SDValue	lowerShuffleAsInsertPS (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
static SDValue	lowerV2F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 2-lane 64-bit floating point shuffles.
static SDValue	lowerV2I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 2-lane 64-bit integer shuffles.
static SDValue	lowerShuffleWithSHUFPS (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
	Lower a vector shuffle using the SHUFPS instruction.
static SDValue	lowerV4F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower 4-lane 32-bit floating point shuffles.
static SDValue	lowerV4I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower 4-lane i32 vector shuffles.
static SDValue	lowerV8I16GeneralSingleInputShuffle (const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.
static SDValue	lowerShuffleAsBlendOfPSHUFBs (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse)
	Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used.
static SDValue	lowerV8I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Generic lowering of 8-lane i16 shuffles.
static SDValue	lowerV8F16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower 8-lane 16-bit floating point shuffles.
static SDValue	lowerShuffleWithPERMV (const SDLoc &DL, MVT VT, ArrayRef< int > OriginalMask, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerV16I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Generic lowering of v16i8 shuffles.
static SDValue	lower128BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Dispatching routine to lower various 128-bit x86 vector shuffles.
static SDValue	splitAndLowerShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool SimpleOnly)
	Generic routine to split vector shuffle into half-sized shuffles.
static SDValue	lowerShuffleAsSplitOrBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Either split a vector in halves or decompose the shuffles and the blend/unpack.
static SDValue	lowerShuffleAsLanePermuteAndSHUFP (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
static SDValue	lowerShuffleAsLanePermuteAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane permutation.
static void	computeInLaneShuffleMask (const ArrayRef< int > &Mask, int LaneSize, SmallVector< int > &InLaneMask)
	Helper to get compute inlane shuffle mask for a complete shuffle mask.
static SDValue	lowerShuffleAsLanePermuteAndShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one source with a lane permutation.
static SDValue	lowerV2X128Shuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering 2-lane 128-bit shuffles.
static SDValue	lowerShuffleAsLanePermuteAndRepeatedMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.
static bool	getHalfShuffleMask (ArrayRef< int > Mask, MutableArrayRef< int > HalfMask, int &HalfIdx1, int &HalfIdx2)
	If the input shuffle mask results in a vector that is undefined in all upper or lower half elements and that mask accesses only 2 halves of the shuffle's operands, return true.
static SDValue	getShuffleHalfVectors (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > HalfMask, int HalfIdx1, int HalfIdx2, bool UndefLower, SelectionDAG &DAG, bool UseConcat=false)
	Given the output values from getHalfShuffleMask(), create a half width shuffle of extracted vectors followed by an insert back to full width.
static SDValue	lowerShuffleWithUndefHalf (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
static SDValue	lowerShuffleAsRepeatedMaskAndLanePermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes.
static bool	matchShuffleWithSHUFPD (MVT VT, SDValue &V1, SDValue &V2, bool &ForceV1Zero, bool &ForceV2Zero, unsigned &ShuffleImm, ArrayRef< int > Mask, const APInt &Zeroable)
static SDValue	lowerShuffleWithSHUFPD (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerShuffleAsVTRUNCAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
static SDValue	lowerShufflePairAsUNPCKAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
static SDValue	lowerV4F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 4-lane 64-bit floating point shuffles.
static SDValue	lowerV4I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 4-lane 64-bit integer shuffles.
static SDValue	lowerV8F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 8-lane 32-bit floating point shuffles.
static SDValue	lowerV8I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 8-lane 32-bit integer shuffles.
static SDValue	lowerV16I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 16-lane 16-bit integer shuffles.
static SDValue	lowerV32I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 32-lane 8-bit integer shuffles.
static SDValue	lower256BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	High-level routine to lower various 256-bit x86 vector shuffles.
static SDValue	lowerV4X128Shuffle (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a vector shuffle as a 128-bit shuffles.
static SDValue	lowerV8F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 8-lane 64-bit floating point shuffles.
static SDValue	lowerV16F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 16-lane 32-bit floating point shuffles.
static SDValue	lowerV8I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 8-lane 64-bit integer shuffles.
static SDValue	lowerV16I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 16-lane 32-bit integer shuffles.
static SDValue	lowerV32I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 32-lane 16-bit integer shuffles.
static SDValue	lowerV64I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle lowering of 64-lane 8-bit integer shuffles.
static SDValue	lower512BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	High-level routine to lower various 512-bit x86 vector shuffles.
static SDValue	lower1BitShuffleAsKSHIFTR (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static int	match1BitShuffleAsKSHIFT (unsigned &Opcode, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable)
static SDValue	lower1BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static bool	canonicalizeShuffleMaskWithCommute (ArrayRef< int > Mask)
	Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.
static bool	canCombineAsMaskOperation (SDValue V, const X86Subtarget &Subtarget)
static SDValue	canonicalizeShuffleMaskWithHorizOp (MutableArrayRef< SDValue > Ops, MutableArrayRef< int > Mask, unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerVECTOR_SHUFFLE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Top-level lowering for x86 vector shuffles.
static SDValue	lowerVECTOR_COMPRESS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerVSELECTtoVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to lower a VSELECT instruction to a vector shuffle.
static SDValue	LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG)
static SDValue	ExtractBitFromMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Extract one bit from mask vector, like v16i1 or v8i1.
static APInt	getExtractedDemandedElts (SDNode *N)
static SDValue	InsertBitToMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Insert one bit to mask vector, like v16i1 or v8i1.
static SDValue	LowerSCALAR_TO_VECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	GetTLSADDR (SelectionDAG &DAG, GlobalAddressSDNode *GA, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LoadGlobalBaseReg=false, bool LocalDynamic=false)
static SDValue	LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
static SDValue	LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
static SDValue	LowerToTLSGeneralDynamicModelX32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
static SDValue	LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool Is64Bit, bool Is64BitLP64)
static SDValue	LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC)
static SDValue	LowerShiftParts (SDValue Op, SelectionDAG &DAG)
	Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount.
static SDValue	LowerI64IntToFP_AVX512DQ (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerI64IntToFP16 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static bool	useVectorCast (unsigned Opcode, MVT FromVT, MVT ToVT, const X86Subtarget &Subtarget)
static SDValue	vectorizeExtractedCast (SDValue Cast, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Given a scalar cast operation that is extracted from a vector, try to vectorize the cast op followed by extraction.
static SDValue	lowerFPToIntToFP (SDValue CastToFP, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Given a scalar cast to FP with a cast to integer operand (almost an ftrunc), try to vectorize the cast ops.
static SDValue	lowerINT_TO_FP_vXi64 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	promoteXINT_TO_FP (SDValue Op, const SDLoc &dl, SelectionDAG &DAG)
static bool	isLegalConversion (MVT VT, bool IsSigned, const X86Subtarget &Subtarget)
static bool	shouldUseHorizontalOp (bool IsSingleSource, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Horizontal vector math instructions may be slower than normal math with shuffles.
static SDValue	LowerUINT_TO_FP_i64 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	64-bit unsigned integer to double expansion.
static SDValue	LowerUINT_TO_FP_i32 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	32-bit unsigned integer to float expansion.
static SDValue	lowerUINT_TO_FP_v2i32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerUINT_TO_FP_vXi32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerUINT_TO_FP_vec (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerAVXExtend (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	SplitAndExtendv16i1 (unsigned ExtOpc, MVT VT, SDValue In, const SDLoc &dl, SelectionDAG &DAG)
static SDValue	LowerZERO_EXTEND_Mask (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerZERO_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	truncateVectorWithPACK (unsigned Opcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Helper to recursively truncate vector elements in half with PACKSS/PACKUS.
static SDValue	truncateVectorWithPACKUS (EVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Truncate using inreg zero extension (AND mask) and X86ISD::PACKUS.
static SDValue	truncateVectorWithPACKSS (EVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Truncate using inreg sign extension and X86ISD::PACKSS.
static SDValue	matchTruncateWithPACK (unsigned &PackOpcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Helper to determine if In truncated to DstVT has the necessary signbits / leading zero bits to be truncated with PACKSS / PACKUS, possibly by converting a SRL node to SRA for sign extension.
static SDValue	LowerTruncateVecPackWithSignBits (MVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	This function lowers a vector truncation of 'extended sign-bits' or 'extended zero-bits' values.
static SDValue	LowerTruncateVecPack (MVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	This function lowers a vector truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations.
static SDValue	LowerTruncateVecI1 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	expandFP_TO_UINT_SSE (MVT VT, SDValue Src, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerFP16_TO_FP (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerFP_TO_FP16 (SDValue Op, SelectionDAG &DAG)
static SDValue	lowerAddSubToHorizontalOp (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation.
static SDValue	LowerFROUND (SDValue Op, SelectionDAG &DAG)
	ISD::FROUND is defined to round to nearest with ties rounding away from 0.
static SDValue	LowerFABSorFNEG (SDValue Op, SelectionDAG &DAG)
	The only differences between FABS and FNEG are the mask and the logic op.
static SDValue	LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerFGETSIGN (SDValue Op, SelectionDAG &DAG)
static SDValue	getBT (SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG)
	Helper for attempting to create a X86ISD::BT node.
static SDValue	getSETCC (X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl, SelectionDAG &DAG)
	Helper for creating a X86ISD::SETCC node.
static bool	isOrXorXorTree (SDValue X, bool Root=true)
	Recursive helper for combineVectorSizedSetCCEquality() to see if we have a recognizable memcmp expansion.
template<typename F >
static SDValue	emitOrXorXorTree (SDValue X, const SDLoc &DL, SelectionDAG &DAG, EVT VecVT, EVT CmpVT, bool HasPT, F SToV)
	Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp expansion.
static SDValue	combineVectorSizedSetCCEquality (EVT VT, SDValue X, SDValue Y, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Try to map a 128-bit or larger integer comparison to vector instructions before type legalization splits it up into chunks.
static bool	matchScalarReduction (SDValue Op, ISD::NodeType BinOp, SmallVectorImpl< SDValue > &SrcOps, SmallVectorImpl< APInt > *SrcMask=nullptr)
	Helper for matching BINOP(EXTRACTELT(X,0),BINOP(EXTRACTELT(X,1),...)) style scalarized (associative) reduction patterns.
static SDValue	LowerVectorAllEqual (const SDLoc &DL, SDValue LHS, SDValue RHS, ISD::CondCode CC, const APInt &OriginalMask, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC)
static SDValue	MatchVectorAllEqualTest (SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC)
static bool	hasNonFlagsUse (SDValue Op)
	return true if Op has a use that doesn't just read flags.
static bool	isProfitableToUseFlagOp (SDValue Op)
static SDValue	EmitTest (SDValue Op, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Emit nodes that will be selected as "test Op0,Op0", or something equivalent.
static SDValue	EmitCmp (SDValue Op0, SDValue Op1, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Emit nodes that will be selected as "cmp Op0,Op1", or something equivalent.
static SDValue	LowerAndToBT (SDValue And, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, X86::CondCode &X86CC)
	Result of 'and' is compared against zero.
static bool	cheapX86FSETCC_SSE (ISD::CondCode SetCCOpcode)
static unsigned	translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1, bool &IsAlwaysSignaling)
	Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs.
static SDValue	splitVSETCC (EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SelectionDAG &DAG, const SDLoc &dl)
	Break a VSETCC 256/512-bit vector into two new 128/256 ones and then concatenate the result back.
static SDValue	LowerIntVSETCC_AVX512 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG)
static SDValue	incDecVectorConstant (SDValue V, SelectionDAG &DAG, bool IsInc, bool NSW)
	Given a buildvector constant, return a new vector constant with each element incremented or decremented.
static SDValue	LowerVSETCCWithSUBUS (SDValue Op0, SDValue Op1, MVT VT, ISD::CondCode Cond, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	As another special case, use PSUBUS[BW] when it's profitable.
static SDValue	LowerVSETCC (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	EmitAVX512Test (SDValue Op0, SDValue Op1, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue &X86CC)
static std::pair< SDValue, SDValue >	getX86XALUOOp (X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG)
static SDValue	LowerXALUO (SDValue Op, SelectionDAG &DAG)
static bool	isX86LogicalCmp (SDValue Op)
	Return true if opcode is a X86 logical comparison.
static bool	isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG)
static SDValue	LowerSELECTWithCmpZero (SDValue CmpVal, SDValue LHS, SDValue RHS, unsigned X86CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerSIGN_EXTEND_Mask (SDValue Op, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerANY_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerEXTEND_VECTOR_INREG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerSIGN_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	splitVectorStore (StoreSDNode *Store, SelectionDAG &DAG)
	Change a vector store into a pair of half-size vector stores.
static SDValue	scalarizeVectorStore (StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
	Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.
static SDValue	LowerStore (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerLoad (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static bool	isAndOrOfSetCCs (SDValue Op, unsigned &Opc)
	Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR.
static SDValue	LowerVACOPY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static unsigned	getTargetVShiftUniformOpcode (unsigned Opc, bool IsVariable)
static SDValue	getTargetVShiftByConstNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG)
	Handle vector element shifts where the shift amount is a constant.
static SDValue	getTargetVShiftNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, SDValue ShAmt, int ShAmtIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Handle vector element shifts by a splat shift amount.
static SDValue	getScalarMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Creates an SDNode for a predicated scalar operation.
static int	getSEHRegistrationNodeSize (const Function *Fn)
static SDValue	recoverFramePointer (SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP)
	When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP.
static SDValue	getAVX2GatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
static SDValue	getGatherNode (SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
static SDValue	getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
static SDValue	getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
static SDValue	expandIntrinsicWChainHelper (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, unsigned TargetOpcode, unsigned SrcReg, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
	Handles the lowering of builtin intrinsics with chain that return their value into registers EDX:EAX.
static void	getReadTimeStampCounter (SDNode *N, const SDLoc &DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
	Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp).
static SDValue	LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	MarkEHRegistrationNode (SDValue Op, SelectionDAG &DAG)
static SDValue	MarkEHGuard (SDValue Op, SelectionDAG &DAG)
static SDValue	EmitTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
	Emit Truncating Store with signed or unsigned saturation.
static SDValue	EmitMaskedTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
	Emit Masked Truncating Store with signed or unsigned saturation.
static SDValue	LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG)
static SDValue	createSetFPEnvNodes (SDValue Ptr, SDValue Chain, const SDLoc &DL, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG, const X86Subtarget &Subtarget)
uint64_t	getGFNICtrlImm (unsigned Opcode, unsigned Amt=0)
SDValue	getGFNICtrlMask (unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, MVT VT, unsigned Amt=0)
static SDValue	LowerVectorCTLZ_AVX512CDI (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Lower a vector CTLZ using native supported vector CTLZ instruction.
static SDValue	LowerVectorCTLZInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerVectorCTLZ (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerCTLZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerCTTZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerAddSub (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerADDSAT_SUBSAT (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerABS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerAVG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMINMAX (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerFMINIMUM_FMAXIMUM (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerABD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMUL (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowervXi8MulWithUNPCK (SDValue A, SDValue B, const SDLoc &dl, MVT VT, bool IsSigned, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue *Low=nullptr)
static SDValue	LowerMULH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMULO (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static bool	supportedVectorShiftWithImm (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
static bool	supportedVectorShiftWithBaseAmnt (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
static bool	supportedVectorVarShift (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
static SDValue	LowerShiftByScalarImmediate (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerShiftByScalarVariable (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	convertShiftLeftToScale (SDValue Amt, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerFunnelShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerRotate (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static std::pair< Value *, BitTestKind >	FindSingleBitChange (Value *V)
static bool	shouldExpandCmpArithRMWInIR (AtomicRMWInst *AI)
static SDValue	emitLockedStackOp (SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue Chain, const SDLoc &DL)
	Emit a locked operation on a stack location which does not change any memory location, but does involve a lock prefix.
static SDValue	LowerATOMIC_FENCE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerCMP_SWAP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	getPMOVMSKB (const SDLoc &DL, SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerBITCAST (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerHorizontalByteSum (SDValue V, MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Compute the horizontal sum of bytes in V for the elements of VT.
static SDValue	LowerVectorCTPOPInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerVectorCTPOP (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerCTPOP (SDValue N, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerBITREVERSE_XOP (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerBITREVERSE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerPARITY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerAtomicArithWithLOCK (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerAtomicArith (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Lower atomic_load_ops into LOCK-prefixed operations.
static SDValue	LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	LowerADDSUBO_CARRY (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerFSINCOS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	ExtendToType (SDValue InOp, MVT NVT, SelectionDAG &DAG, bool FillWithZeroes=false)
	Widen a vector input to a vector of NVT.
static SDValue	LowerMSCATTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMLOAD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMSTORE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerMGATHER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerADDRSPACECAST (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerCVTPS2PH (SDValue Op, SelectionDAG &DAG)
static SDValue	LowerPREFETCH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
static SDValue	LowerFCanonicalize (SDValue Op, SelectionDAG &DAG)
static StringRef	getInstrStrFromOpNo (const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo)
static SDValue	getFlagsOfCmpZeroFori1 (SelectionDAG &DAG, const SDLoc &DL, SDValue Mask)
static bool	isEFLAGSLiveAfter (MachineBasicBlock::iterator Itr, MachineBasicBlock *BB)
static MachineBasicBlock *	emitXBegin (MachineInstr &MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII)
	Utility function to emit xbegin specifying the start of an RTM region.
static bool	checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
static bool	isCMOVPseudo (MachineInstr &MI)
static MachineInstrBuilder	createPHIsForCMOVsInSinkBB (MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB, MachineBasicBlock *SinkMBB)
static unsigned	getSUBriOpcode (bool IsLP64)
static unsigned	getOpcodeForIndirectThunk (unsigned RPOpc)
static const char *	getIndirectThunkSymbol (const X86Subtarget &Subtarget, unsigned Reg)
static void	computeKnownBitsForPSADBW (SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth)
static void	computeKnownBitsForPMADDWD (SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth)
static void	computeKnownBitsForPMADDUBSW (SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth)
static KnownBits	computeKnownBitsForHorizontalOperation (const SDValue Op, const APInt &DemandedElts, unsigned Depth, const SelectionDAG &DAG, const function_ref< KnownBits(const KnownBits &, const KnownBits &)> KnownBitsFunc)
static SDValue	narrowLoadToVZLoad (LoadSDNode *LN, MVT MemVT, MVT VT, SelectionDAG &DAG)
static bool	matchUnaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue V1, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT)
static bool	matchUnaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
static bool	matchBinaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT, bool IsUnary)
static bool	matchBinaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
static SDValue	combineX86ShuffleChainWithExtract (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineX86ShuffleChain (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Combine an arbitrary chain of shuffles into a single instruction if possible.
static SDValue	combineX86ShufflesConstants (MVT VT, ArrayRef< SDValue > Ops, ArrayRef< int > Mask, bool HasVariableMask, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	combineX86ShufflesRecursively (ArrayRef< SDValue > SrcOps, int SrcOpIndex, SDValue Root, ArrayRef< int > RootMask, ArrayRef< const SDNode * > SrcNodes, unsigned Depth, unsigned MaxDepth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Fully generic combining of x86 shuffle instructions.
static SDValue	combineX86ShufflesRecursively (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Helper entry wrapper to combineX86ShufflesRecursively.
static SmallVector< int, 4 >	getPSHUFShuffleMask (SDValue N)
	Get the PSHUF-style mask from PSHUF node.
static SDValue	combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
	Search for a combinable shuffle across a chain ending in pshufd.
static SDValue	combineCommutableSHUFP (SDValue N, MVT VT, const SDLoc &DL, SelectionDAG &DAG)
static SDValue	combineBlendOfPermutes (MVT VT, SDValue N0, SDValue N1, ArrayRef< int > BlendMask, const APInt &DemandedElts, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
static bool	isUnaryOp (unsigned Opcode)
static SDValue	canonicalizeShuffleWithOp (SDValue N, SelectionDAG &DAG, const SDLoc &DL)
static SDValue	canonicalizeLaneShuffleWithRepeatedOps (SDValue V, SelectionDAG &DAG, const SDLoc &DL)
	Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()).
static SDValue	combineTargetShuffle (SDValue N, const SDLoc &DL, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Try to combine x86 target specific shuffles.
static bool	isAddSubOrSubAddMask (ArrayRef< int > Mask, bool &Op0Even)
	Checks if the shuffle mask takes subsequent elements alternately from two vectors.
static bool	isAddSubOrSubAdd (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, bool &IsSubAdd)
	Returns true iff the shuffle node N can be replaced with ADDSUB(SUBADD) operation.
static SDValue	combineShuffleToFMAddSub (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.
static SDValue	combineShuffleToAddSubOrFMAddSub (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
	Try to combine a shuffle into a target-specific add-sub or mul-add-sub node.
static SDValue	combineShuffleOfConcatUndef (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	narrowShuffle (ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
	If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the low half of each source vector and does not set any high half elements in the destination vector, narrow the shuffle to half its original size.
static SDValue	combineShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static bool	checkBitcastSrcVectorSize (SDValue Src, unsigned Size, bool AllowTruncate)
static unsigned	getAltBitOpcode (unsigned Opcode)
static SDValue	adjustBitcastSrcVectorSSE1 (SelectionDAG &DAG, SDValue Src, const SDLoc &DL)
static SDValue	signExtendBitcastSrcVector (SelectionDAG &DAG, EVT SExtVT, SDValue Src, const SDLoc &DL)
static SDValue	combineBitcastvxi1 (SelectionDAG &DAG, EVT VT, SDValue Src, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	combinevXi1ConstantToInteger (SDValue Op, SelectionDAG &DAG)
static SDValue	combineCastedMaskArithmetic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	createMMXBuildVector (BuildVectorSDNode *BV, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineBitcastToBoolVector (EVT VT, SDValue V, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned Depth=0)
static SDValue	combineBitcast (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static bool	detectExtMul (SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0, SDValue &Op1)
static bool	detectZextAbsDiff (const SDValue &Abs, SDValue &Op0, SDValue &Op1)
static SDValue	createVPDPBUSD (SelectionDAG &DAG, SDValue LHS, SDValue RHS, unsigned &LogBias, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	createPSADBW (SelectionDAG &DAG, const SDValue &Zext0, const SDValue &Zext1, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	combineMinMaxReduction (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combinePredicateReduction (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineVPDPBUSDPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineBasicSADPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineExtractFromVectorLoad (SDNode *N, EVT VecVT, SDValue SrcVec, uint64_t Idx, const SDLoc &dl, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineExtractWithShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	scalarizeExtEltFP (SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget, TargetLowering::DAGCombinerInfo &DCI)
	Extracting a scalar FP value from vector element 0 is free, so extract each operand first, then perform the math as a scalar op.
static SDValue	combineArithReduction (SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Try to convert a vector reduction sequence composed of binops and shuffles into horizontal ops.
static SDValue	combineExtractVectorElt (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.
static SDValue	combineToExtendBoolVectorInReg (unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVSelectWithAllOnesOrZeros (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation.
static SDValue	narrowVectorSelect (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
	If both arms of a vector select are concatenated vectors, split the select, and concatenate the result to eliminate a wide (256-bit) vector instruction: vselect Cond, (concat T0, T1), (concat F0, F1) --> concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1)
static SDValue	combineSelectOfTwoConstants (SDNode *N, SelectionDAG &DAG, const SDLoc &DL)
static SDValue	combineVSelectToBLENDV (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element.
static SDValue	combineLogicBlendIntoConditionalNegate (EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	commuteSelect (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	combineSelect (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Do target-specific dag combines on SELECT and VSELECT nodes.
static SDValue	combineSetCCAtomicArith (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction.
static SDValue	checkSignTestSetCCCombine (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG)
static SDValue	checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC)
static bool	checkBoolTestAndOrSetCCCombine (SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd)
	Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
static SDValue	combineCarryThroughADD (SDValue EFLAGS, SelectionDAG &DAG)
static SDValue	combinePTESTCC (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	If we are inverting an PTEST/TESTP operand, attempt to adjust the CC to avoid the inversion.
static SDValue	combineSetCCMOVMSK (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineSetCCEFLAGS (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values.
static SDValue	combineCMov (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].
static bool	canReduceVMulWidth (SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode)
static SDValue	reduceVMULWidth (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code.
static SDValue	combineMulSpecial (uint64_t MulAmt, SDNode *N, SelectionDAG &DAG, EVT VT, const SDLoc &DL)
static SDValue	combineMulToPMADDWD (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineMulToPMULDQ (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineMul (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineShiftToPMULH (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget)
static SDValue	combineShiftLeft (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineShiftRightArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineShiftRightLogical (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineHorizOpWithShuffle (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineVectorPack (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVectorHADDSUB (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVectorShiftVar (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVectorShiftImm (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVectorInsert (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineCompareEqual (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends.
static SDValue	combineAndNotIntoANDNP (SDNode *N, SelectionDAG &DAG)
	Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
static SDValue	combineAndShuffleNot (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Try to fold: and (vector_shuffle<Z,...,Z> (insert_vector_elt undef, (xor X, -1), Z), undef), Y -> andnp (vector_shuffle<Z,...,Z> (insert_vector_elt undef, X, Z), undef), Y.
static SDValue	PromoteMaskArithmetic (SDValue N, const SDLoc &DL, EVT VT, SelectionDAG &DAG, unsigned Depth)
static SDValue	PromoteMaskArithmetic (SDValue N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static unsigned	convertIntLogicToFPLogicOpcode (unsigned Opcode)
static SDValue	convertIntLogicToFPLogic (unsigned Opc, const SDLoc &DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	If both input operands of a logic op are being cast from floating-point types or FP compares, try to convert this into a floating-point logic node to avoid unnecessary moves from SSE to integer registers.
static SDValue	combineBitOpWithMOVMSK (unsigned Opc, const SDLoc &DL, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue	combineBitOpWithShift (unsigned Opc, const SDLoc &DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue	combineBitOpWithPACK (unsigned Opc, const SDLoc &DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue	combineAndMaskToShift (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	If this is a zero/all-bits result that is bitwise-anded with a low bits mask.
static SDValue	getIndexFromUnindexedLoad (LoadSDNode *Ld)
static bool	hasBZHI (const X86Subtarget &Subtarget, MVT VT)
static SDValue	combineAndNotOrIntoAndNotAnd (SDNode *N, SelectionDAG &DAG)
	Folds (and X, (or Y, ~Z)) --> (and X, ~(and ~Y, Z)) This undoes the inverse fold performed in InstCombine.
static SDValue	combineAndLoadToBZHI (SDNode *Node, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineScalarAndWithMaskSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	getBMIMatchingOp (unsigned Opc, SelectionDAG &DAG, SDValue OpMustEq, SDValue Op, unsigned Depth)
static SDValue	combineBMILogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineX86SubCmpForFlags (SDNode *N, SDValue Flag, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST)
static SDValue	combineAndOrForCcmpCtest (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST)
static SDValue	combineAnd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	canonicalizeBitSelect (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static bool	matchLogicBlend (SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask)
static SDValue	combineLogicBlendIntoPBLENDV (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	lowerX86CmpEqZeroToCtlzSrl (SDValue Op, SelectionDAG &DAG)
static SDValue	combineOrCmpEqZeroToCtlzSrl (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	foldMaskedMergeImpl (SDValue And0_L, SDValue And0_R, SDValue And1_L, SDValue And1_R, const SDLoc &DL, SelectionDAG &DAG)
static SDValue	foldMaskedMerge (SDNode *Node, SelectionDAG &DAG)
	Fold "masked merge" expressions like (m & x) | (~m & y) into the equivalent ((x ^ y) & m) ^ y) pattern.
static SDValue	combineAddOrSubToADCOrSBB (bool IsSub, const SDLoc &DL, EVT VT, SDValue X, SDValue Y, SelectionDAG &DAG, bool ZeroSecondOpOnly=false)
	If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.
static SDValue	combineAddOrSubToADCOrSBB (SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
	If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.
static SDValue	combineOrXorWithSETCC (unsigned Opc, const SDLoc &DL, EVT VT, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue	combineOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	foldXorTruncShiftIntoCmp (SDNode *N, SelectionDAG &DAG)
	Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1)
static SDValue	foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1.
static SDValue	detectUSatPattern (SDValue In, EVT VT, SelectionDAG &DAG, const SDLoc &DL)
	Detect patterns of truncation with unsigned saturation:
static SDValue	detectSSatPattern (SDValue In, EVT VT, bool MatchPackUS=false)
	Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type).
static SDValue	combineTruncateWithSat (SDValue In, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineConstantPoolLoads (SDNode *N, const SDLoc &dl, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static int	getOneTrueElt (SDValue V)
	If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element.
static bool	getParamsForOneTrueMaskedElt (MaskedLoadStoreSDNode *MaskedOp, SelectionDAG &DAG, SDValue &Addr, SDValue &Index, Align &Alignment, unsigned &Offset)
	Given a masked memory load/store operation, return true if it has one mask bit set.
static SDValue	reduceMaskedLoadToScalarLoad (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert.
static SDValue	combineMaskedLoadConstantMask (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineMaskedLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	reduceMaskedStoreToScalarStore (MaskedStoreSDNode *MS, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store.
static SDValue	combineMaskedStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVEXTRACT_STORE (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static bool	isHorizontalBinOp (unsigned HOpcode, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsCommutative, SmallVectorImpl< int > &PostShuffleMask, bool ForceHorizOp)
	Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS.
static SDValue	combineToHorizontalAddSub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineFMulcFCMulc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineFaddCFmul (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineFaddFsub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Do target-specific dag combines on floating-point adds/subs.
static SDValue	combineLRINT_LLRINT (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineTruncatedArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
	Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen.
static SDValue	combinePMULH (SDValue Src, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	detectPMADDUBSW (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
static SDValue	combineTruncate (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineVTRUNC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	isFNEG (SelectionDAG &DAG, SDNode *N, unsigned Depth=0)
	Returns the negated value if the node N flips sign of FP value.
static unsigned	negateFMAOpcode (unsigned Opcode, bool NegMul, bool NegAcc, bool NegRes)
static SDValue	combineFneg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Do target-specific dag combines on floating point negations.
static SDValue	lowerX86FPLogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	foldXor1SetCC (SDNode *N, SelectionDAG &DAG)
	Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
static SDValue	combineXorSubCTLZ (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineXor (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineBITREVERSE (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineAVG (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineBEXTR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static bool	isNullFPScalarOrVectorConst (SDValue V)
static SDValue	getNullFPConstForNullVal (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value.
static SDValue	combineFAndFNotToFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineFAnd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Do target-specific dag combines on X86ISD::FAND nodes.
static SDValue	combineFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Do target-specific dag combines on X86ISD::FANDN nodes.
static SDValue	combineFOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
static SDValue	combineFMinFMax (SDNode *N, SelectionDAG &DAG)
	Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
static SDValue	combineFMinNumFMaxNum (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineX86INT_TO_FP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineCVTP2I_CVTTP2I (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineAndnp (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Do target-specific dag combines on X86ISD::ANDNP nodes.
static SDValue	combineBT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineCVTPH2PS (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineSextInRegCmov (SDNode *N, SelectionDAG &DAG)
static SDValue	combineSignExtendInReg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	promoteExtBeforeAdd (SDNode *Ext, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	sext(add_nsw(x, C)) --> add(sext(x), C_sext) zext(add_nuw(x, C)) --> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode.
static SDValue	combineToExtendCMOV (SDNode *Extend, SelectionDAG &DAG)
static SDValue	combineExtSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineSext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	getInvertedVectorForFMA (SDValue V, SelectionDAG &DAG)
static SDValue	combineFMA (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineFMADDSUB (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineZext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	truncateAVX512SetCCNoBWI (EVT VT, EVT OpVT, SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just pre-promote its result type since vXi1 vectors don't get promoted during type legalization.
static SDValue	combineSetCC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineMOVMSK (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineTESTP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineX86GatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	rebuildGatherScatter (MaskedGatherScatterSDNode *GorS, SDValue Index, SDValue Base, SDValue Scale, SelectionDAG &DAG)
static SDValue	combineGatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineX86SetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineBrCond (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	Optimize branch condition evaluation.
static SDValue	combineVectorCompareAndMaskUnaryOp (SDNode *N, SelectionDAG &DAG)
static SDValue	combineToFPTruncExtElt (SDNode *N, SelectionDAG &DAG)
	If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector element with a bitcast.
static SDValue	combineUIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineSIntToFP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static bool	needCarryOrOverflowFlag (SDValue Flags)
static bool	onlyZeroFlagUsed (SDValue Flags)
static SDValue	combineCMP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineX86AddSub (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST)
static SDValue	combineSBB (SDNode *N, SelectionDAG &DAG)
static SDValue	combineADC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	matchPMADDWD (SelectionDAG &DAG, SDNode *N, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
static SDValue	matchPMADDWD_2 (SelectionDAG &DAG, SDNode *N, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
static SDValue	combineAddOfPMADDWD (SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, EVT VT)
static SDValue	pushAddIntoCmovOfConsts (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
	CMOV of constants requires materializing constant operands in registers.
static SDValue	combineAdd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineSubABS (EVT VT, const SDLoc &DL, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue	combineSubSetcc (SDNode *N, SelectionDAG &DAG)
static SDValue	combineX86CloadCstore (SDNode *N, SelectionDAG &DAG)
static SDValue	combineSub (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVectorCompare (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static std::optional< unsigned >	CastIntSETCCtoFP (MVT VT, ISD::CondCode CC, unsigned NumSignificantBitsLHS, unsigned NumSignificantBitsRHS)
static SDValue	combineConcatVectorOps (const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
	Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.
static SDValue	combineCONCAT_VECTORS (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineINSERT_SUBVECTOR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	narrowExtractedVectorSelect (SDNode *Ext, const SDLoc &DL, SelectionDAG &DAG)
	If we are extracting a subvector of a vector select and the select condition is composed of concatenated vectors, try to narrow the select width.
static SDValue	combineEXTRACT_SUBVECTOR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineScalarToVector (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combinePMULDQ (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineVPMADD (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineEXTEND_VECTOR_INREG (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineKSHIFT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineFP16_TO_FP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineFP_EXTEND (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
static SDValue	combineBROADCAST_LOAD (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineFP_ROUND (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
static SDValue	combineMOVDQ2Q (SDNode *N, SelectionDAG &DAG)
static SDValue	combinePDEP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	FixupMMXIntrinsicTypes (SDNode *N, SelectionDAG &DAG)
static SDValue	combineINTRINSIC_WO_CHAIN (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineINTRINSIC_W_CHAIN (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	combineINTRINSIC_VOID (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static bool	matchAsm (StringRef S, ArrayRef< const char * > Pieces)
static bool	clobbersFlagRegisters (const SmallVector< StringRef, 4 > &AsmPieces)
static X86::CondCode	parseConstraintCode (llvm::StringRef Constraint)
static bool	isGRClass (const TargetRegisterClass &RC)
	Check if RC is a general purpose register class.
static bool	isFRClass (const TargetRegisterClass &RC)
	Check if RC is a vector register class.
static bool	isVKClass (const TargetRegisterClass &RC)
	Check if RC is a mask register class.
static bool	useEGPRInlineAsm (const X86Subtarget &Subtarget)

Variables
static cl::opt< int >	ExperimentalPrefInnermostLoopAlignment ("x86-experimental-pref-innermost-loop-alignment", cl::init(4), cl::desc("Sets the preferable loop alignment for experiments (as log2 bytes) " "for innermost loops only. If specified, this option overrides " "alignment set by x86-experimental-pref-loop-alignment."), cl::Hidden)
static cl::opt< int >	BrMergingBaseCostThresh ("x86-br-merging-base-cost", cl::init(2), cl::desc("Sets the cost threshold for when multiple conditionals will be merged " "into one branch versus be split in multiple branches. Merging " "conditionals saves branches at the cost of additional instructions. " "This value sets the instruction cost limit, below which conditionals " "will be merged, and above which conditionals will be split. Set to -1 " "to never merge branches."), cl::Hidden)
static cl::opt< int >	BrMergingCcmpBias ("x86-br-merging-ccmp-bias", cl::init(6), cl::desc("Increases 'x86-br-merging-base-cost' in cases that the target " "supports conditional compare instructions."), cl::Hidden)
static cl::opt< bool >	WidenShift ("x86-widen-shift", cl::init(true), cl::desc("Replacte narrow shifts with wider shifts."), cl::Hidden)
static cl::opt< int >	BrMergingLikelyBias ("x86-br-merging-likely-bias", cl::init(0), cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "likely, then it is likely that if the conditionals are split " "both sides will be executed, so it may be desirable to increase " "the instruction cost threshold. Set to -1 to never merge likely " "branches."), cl::Hidden)
static cl::opt< int >	BrMergingUnlikelyBias ("x86-br-merging-unlikely-bias", cl::init(-1), cl::desc("Decreases 'x86-br-merging-base-cost' in cases that it is unlikely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "unlikely, then it is unlikely that if the conditionals are split " "both sides will be executed, so it may be desirable to decrease " "the instruction cost threshold. Set to -1 to never merge unlikely " "branches."), cl::Hidden)
static cl::opt< bool >	MulConstantOptimization ("mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc."), cl::Hidden)
const unsigned	X87StateSize = 28
const unsigned	FPStateSize = 32
const unsigned	FPStateSizeInBits = FPStateSize * 8

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "x86-isel"

Definition at line 68 of file X86ISelLowering.cpp.

GET_EGPR_IF_ENABLED

#define GET_EGPR_IF_ENABLED

(

OPC

)

(Subtarget.hasEGPR() ? OPC##_EVEX : OPC)

NODE_NAME_CASE

#define NODE_NAME_CASE

(

NODE

)

case X86ISD::NODE: return "X86ISD::" #NODE;

Enumeration Type Documentation

BitTestKind

enum BitTestKind : unsigned

Enumerator
UndefBit
ConstantBit
NotConstantBit
ShiftBit
NotShiftBit

Definition at line 31253 of file X86ISelLowering.cpp.

ShrinkMode

enum class ShrinkMode

strong

Different mul shrinking modes.

Enumerator
MULS8
MULU8
MULS16
MULU16

Definition at line 48530 of file X86ISelLowering.cpp.

Function Documentation

adjustBitcastSrcVectorSSE1()

static SDValue adjustBitcastSrcVectorSSE1 ( SelectionDAG &  DAG, SDValue  Src, const SDLoc &  DL  )

static

Definition at line 44423 of file X86ISelLowering.cpp.

References adjustBitcastSrcVectorSSE1(), llvm::ISD::AND, llvm::ISD::BITCAST, DL, getAltBitOpcode(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isNormalLoad(), llvm::ISD::OR, llvm::ISD::SETCC, llvm::ISD::SETLT, and llvm::ISD::XOR.

Referenced by adjustBitcastSrcVectorSSE1(), and combineBitcastvxi1().

buildFromShuffleMostly()

static SDValue buildFromShuffleMostly ( SDValue  Op, const SDLoc &  DL, SelectionDAG &  DAG  )

static

Definition at line 7709 of file X86ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), getUnderlyingExtractedFromVec(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), Idx, llvm::ISD::INSERT_VECTOR_ELT, llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorBase< Size_T >::size(), and llvm::ISD::UNDEF.

canCombineAsMaskOperation()

static bool canCombineAsMaskOperation ( SDValue  V, const X86Subtarget &  Subtarget  )

static

Definition at line 17793 of file X86ISelLowering.cpp.

References llvm::ISD::ABS, llvm::ISD::ADD, llvm::ISD::AND, llvm::X86Subtarget::hasAVX512(), llvm::ISD::MUL, llvm::ISD::OR, llvm::ISD::SHL, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::UMAX, llvm::ISD::UMIN, and llvm::ISD::XOR.

Referenced by commuteSelect(), and lowerVECTOR_SHUFFLE().

canLowerByDroppingElements()

static int canLowerByDroppingElements ( ArrayRef< int >  Mask, bool  MatchEven, bool  IsSingleInput  )

static

Check whether a compaction lowering can be done by dropping even/odd elements and compute how many times even/odd elements must be dropped.

This handles shuffles which take every Nth element where N is a power of two. Example shuffle masks:

(even) N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24

(odd) N = 1: 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31

Any of these lanes can of course be undef.

This routine only supports N <= 3. FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here for larger N.

Returns

N above, or the number of times even/odd elements must be dropped if there is such a number. Otherwise returns zero.

Definition at line 10512 of file X86ISelLowering.cpp.

References assert(), llvm::isPowerOf2_32(), N, and llvm::Offset.

Referenced by lowerV16I8Shuffle(), and lowerV8I16Shuffle().

canonicalizeBitSelect()

static SDValue canonicalizeBitSelect ( SDNode *  N, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 51155 of file X86ISelLowering.cpp.

References A, llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), B, llvm::CallingConv::C, DL, getAVX512Node(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::MVT::isVector(), N, llvm::ISD::OR, llvm::peekThroughBitcasts(), llvm::SmallVectorBase< Size_T >::size(), useVPTERNLOG(), llvm::X86ISD::VPTERNLOG, X, and Y.

Referenced by combineOr().

canonicalizeLaneShuffleWithRepeatedOps()

static SDValue canonicalizeLaneShuffleWithRepeatedOps ( SDValue  V, SelectionDAG &  DAG, const SDLoc &  DL  )

static

Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()).

Definition at line 41561 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::isUndef(), LHS, llvm::X86ISD::MOVDDUP, llvm::peekThroughBitcasts(), llvm::X86ISD::PSHUFD, RHS, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.

Referenced by combineTargetShuffle().

canonicalizeShuffleMaskWithCommute()

static bool canonicalizeShuffleMaskWithCommute ( ArrayRef< int >  Mask )

static

Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.

Definition at line 17731 of file X86ISelLowering.cpp.

References assert(), and Size.

Referenced by combineX86ShuffleChainWithExtract(), combineX86ShufflesRecursively(), and lowerVECTOR_SHUFFLE().

canonicalizeShuffleMaskWithHorizOp()

static SDValue canonicalizeShuffleMaskWithHorizOp ( MutableArrayRef< SDValue >  Ops, MutableArrayRef< int >  Mask, unsigned  RootSizeInBits, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 40264 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::any_of(), assert(), llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::ArrayRef< T >::empty(), extract128BitVector(), llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::SelectionDAG::getBitcast(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86ISD::HADD, llvm::Hi, llvm::X86ISD::HSUB, isInRange(), llvm::SDNode::isOnlyUserOf(), isRepeatedTargetShuffleMask(), isUndefInRange(), isUndefOrZero(), isUndefOrZeroOrInRange(), LHS, llvm::Lo, llvm::M0(), llvm::M1(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, scaleShuffleElements(), shouldUseHorizontalOp(), llvm::X86ISD::SHUFP, llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, std::swap(), and widenSubVector().

Referenced by combineX86ShufflesRecursively(), and lowerVECTOR_SHUFFLE().

canonicalizeShuffleWithOp()

static SDValue canonicalizeShuffleWithOp ( SDValue  N, SelectionDAG &  DAG, const SDLoc &  DL  )

static

Definition at line 41377 of file X86ISelLowering.cpp.

References llvm::X86ISD::BLENDI, llvm::X86ISD::CVTPH2PS, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), getTargetConstantFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_SUBVECTOR, llvm::X86ISD::INSERTPS, llvm::TargetLoweringBase::isBinOp(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), isLogicOp(), isShuffleFoldableLoad(), llvm::SelectionDAG::isSplatValue(), isTargetShuffle(), isUnaryOp(), LHS, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, N, llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, RHS, llvm::X86ISD::SHUFP, llvm::ISD::SINT_TO_FP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.

Referenced by combineShuffle().

canReduceVMulWidth()

static bool canReduceVMulWidth ( SDNode *  N, SelectionDAG &  DAG, ShrinkMode &  Mode  )

static

Definition at line 48532 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::EVT::getScalarSizeInBits(), N, and llvm::SelectionDAG::SignBitIsZero().

Referenced by matchPMADDWD(), and reduceVMULWidth().

canScaleShuffleElements()

static bool canScaleShuffleElements ( ArrayRef< int >  Mask, unsigned  NumDstElts  )

static

Definition at line 3877 of file X86ISelLowering.cpp.

References scaleShuffleElements().

Referenced by combineBlendOfPermutes(), and combineSetCCMOVMSK().

canWidenShuffleElements() [1/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask )

static

Definition at line 3842 of file X86ISelLowering.cpp.

References canWidenShuffleElements().

canWidenShuffleElements() [2/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask, const APInt &  Zeroable, bool  V2IsZero, SmallVectorImpl< int > &  WidenedMask  )

static

Definition at line 3826 of file X86ISelLowering.cpp.

References assert(), canWidenShuffleElements(), llvm::APInt::isZero(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

canWidenShuffleElements() [3/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask, SmallVectorImpl< int > &  WidenedMask  )

static

Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.

Appends the mask for wider elements in WidenedMask if valid. Otherwise leaves it in an unspecified state.

NOTE: This must handle normal vector shuffle masks and target vector shuffle masks. The latter have the special property of a '-2' representing a zero-ed lane of a vector.

Definition at line 3776 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::M0(), llvm::M1(), llvm::SmallVectorBase< Size_T >::size(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by canWidenShuffleElements(), combineBlendOfPermutes(), combineExtractWithShuffle(), combineX86ShuffleChainWithExtract(), combineX86ShufflesRecursively(), LowerShift(), lowerShuffleAsBlendAndPermute(), lowerV2X128Shuffle(), lowerV4X128Shuffle(), lowerVECTOR_SHUFFLE(), and scaleShuffleElements().

CastIntSETCCtoFP()

static std::optional< unsigned > CastIntSETCCtoFP ( MVT  VT, ISD::CondCode  CC, unsigned  NumSignificantBitsLHS, unsigned  NumSignificantBitsRHS  )

static

Definition at line 57159 of file X86ISelLowering.cpp.

References assert(), CC, llvm::MVT::getFltSemantics(), llvm::MVT::getScalarType(), llvm::APFloatBase::semanticsPrecision(), llvm::ISD::SETEQ, llvm::ISD::SETGT, and llvm::ISD::SINT_TO_FP.

Referenced by combineConcatVectorOps().

cheapX86FSETCC_SSE()

static bool cheapX86FSETCC_SSE ( ISD::CondCode  SetCCOpcode )

static

Definition at line 23356 of file X86ISelLowering.cpp.

References llvm::ISD::SETONE, and llvm::ISD::SETUEQ.

Referenced by convertIntLogicToFPLogic(), and LowerVSETCC().

checkAndUpdateEFLAGSKill()

static bool checkAndUpdateEFLAGSKill ( MachineBasicBlock::iterator  SelectItr, MachineBasicBlock *  BB, const TargetRegisterInfo *  TRI  )

static

Definition at line 35563 of file X86ISelLowering.cpp.

References isEFLAGSLiveAfter(), and TRI.

checkBitcastSrcVectorSize()

static bool checkBitcastSrcVectorSize ( SDValue  Src, unsigned  Size, bool  AllowTruncate  )

static

Definition at line 44381 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, checkBitcastSrcVectorSize(), llvm::ISD::FREEZE, llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::OR, llvm::ISD::SELECT, llvm::ISD::SETCC, Size, llvm::ISD::TRUNCATE, llvm::ISD::VSELECT, and llvm::ISD::XOR.

Referenced by checkBitcastSrcVectorSize(), and combineBitcastvxi1().

checkBoolTestAndOrSetCCCombine()

static bool checkBoolTestAndOrSetCCCombine ( SDValue  Cond, X86::CondCode &  CC0, X86::CondCode &  CC1, SDValue &  Flags, bool &  isAnd  )

static

Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.

Match: (X86or (X86setcc) (X86setcc)) (X86cmp (and (X86setcc) (X86setcc)), 0)

Definition at line 47758 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::AND, llvm::X86ISD::CMP, Cond, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::isNullConstant(), llvm::ISD::OR, llvm::X86ISD::OR, and llvm::X86ISD::SETCC.

Referenced by combineCMov().

checkBoolTestSetCCCombine()

static SDValue checkBoolTestSetCCCombine ( SDValue  Cmp, X86::CondCode &  CC  )

static

Definition at line 47640 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::CallingConv::C, CC, llvm::X86ISD::CMOV, llvm::X86ISD::CMP, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::ConstantSDNode::getZExtValue(), llvm::isOneConstant(), llvm::X86ISD::RDRAND, llvm::X86ISD::RDSEED, llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineSetCCEFLAGS().

checkSignTestSetCCCombine()

static SDValue checkSignTestSetCCCombine ( SDValue  Cmp, X86::CondCode &  CC, SelectionDAG &  DAG  )

static

Definition at line 47571 of file X86ISelLowering.cpp.

References llvm::ISD::AND, CC, llvm::X86ISD::CMP, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NS, llvm::X86::COND_S, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getValidShiftAmount(), llvm::isNullConstant(), llvm::APInt::lshrInPlace(), llvm::X86ISD::OR, llvm::ISD::SHL, llvm::SelectionDAG::SignBitIsZero(), and llvm::ISD::SRA.

Referenced by combineSetCCEFLAGS().

clobbersFlagRegisters()

static bool clobbersFlagRegisters ( const SmallVector< StringRef, 4 > &  AsmPieces )

static

Definition at line 59530 of file X86ISelLowering.cpp.

References llvm::is_contained(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by llvm::X86TargetLowering::ExpandInlineAsm().

collectConcatOps()

static bool collectConcatOps ( SDNode *  N, SmallVectorImpl< SDValue > &  Ops, SelectionDAG &  DAG  )

static

Definition at line 4155 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::append(), assert(), collectConcatOps(), llvm::ISD::CONCAT_VECTORS, llvm::SmallVectorBase< Size_T >::empty(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::Hi, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::isNullConstant(), llvm::Lo, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by collectConcatOps(), combineBitcastvxi1(), combineINSERT_SUBVECTOR(), combineSetCCMOVMSK(), combineTargetShuffle(), isFreeToSplitVector(), IsNOT(), and isUpperSubvectorUndef().

combineADC()

static SDValue combineADC ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 56466 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADC, llvm::ISD::ADD, llvm::ISD::AND, combineCarryThroughADD(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::X86::COND_B, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), LHS, N, RHS, and llvm::X86ISD::SETCC_CARRY.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineAdd()

static SDValue combineAdd ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 56858 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADC, llvm::ISD::ADD, assert(), combineAddOfPMADDWD(), combineAddOrSubToADCOrSBB(), combineToHorizontalAddSub(), llvm::ISD::CONCAT_VECTORS, concatSubVectors(), DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDNode::getVTList(), getZeroVector(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDNode::hasOneUse(), llvm::ISD::isBuildVectorAllZeros(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86::isZeroNode(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_BinOp(), llvm::PatternMatch::m_Value(), matchPMADDWD(), matchPMADDWD_2(), N, llvm::X86ISD::PSADBW, pushAddIntoCmovOfConsts(), Select, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::X86Subtarget::useAVX512Regs(), llvm::X86ISD::VPDPWSSD, llvm::X86ISD::VPMADDWD, llvm::SelectionDAG::willNotOverflowAdd(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineAddOfPMADDWD()

static SDValue combineAddOfPMADDWD ( SelectionDAG &  DAG, SDValue  N0, SDValue  N1, const SDLoc &  DL, EVT  VT  )

static

Definition at line 56749 of file X86ISelLowering.cpp.

References DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::APInt::getSplat(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), LHS, llvm::SelectionDAG::MaskedValueIsZero(), RHS, and llvm::X86ISD::VPMADDWD.

Referenced by combineAdd().

combineAddOrSubToADCOrSBB() [1/2]

static SDValue combineAddOrSubToADCOrSBB ( bool  IsSub, const SDLoc &  DL, EVT  VT, SDValue  X, SDValue  Y, SelectionDAG &  DAG, bool  ZeroSecondOpOnly = false  )

static

If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.

This replaces TEST+SET+{ADD/SUB} with CMP+{ADC, SBB}. Also try (ADD/SUB)+(AND(SRL,1)) bit extraction pattern with BT+{ADC, SBB}.

Definition at line 51465 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADC, llvm::ISD::AND, CC, llvm::X86ISD::CMP, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getVTList(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::EVT::isInteger(), llvm::isOneConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86::isZeroNode(), LowerAndToBT(), llvm::X86ISD::SBB, llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::ISD::SETNE, llvm::X86ISD::SUB, X, Y, and llvm::ISD::ZERO_EXTEND.

Referenced by combineAdd(), combineAddOrSubToADCOrSBB(), combineOrXorWithSETCC(), combineSub(), and combineX86AddSub().

combineAddOrSubToADCOrSBB() [2/2]

static SDValue combineAddOrSubToADCOrSBB ( SDNode *  N, const SDLoc &  DL, SelectionDAG &  DAG  )

static

If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.

This replaces TEST+SET+{ADD/SUB} with CMP+{ADC, SBB}.

Definition at line 51649 of file X86ISelLowering.cpp.

References combineAddOrSubToADCOrSBB(), DL, llvm::SelectionDAG::getNegative(), N, llvm::ISD::SUB, X, and Y.

combineAnd()

static SDValue combineAnd ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 50885 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::all_of(), llvm::ISD::AND, assert(), llvm::APInt::clearAllBits(), combineAndLoadToBZHI(), combineAndMaskToShift(), combineAndNotIntoANDNP(), combineAndNotOrIntoAndNotAnd(), combineAndOrForCcmpCtest(), combineAndShuffleNot(), combineBitcastvxi1(), combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineBMILogicOp(), combineCompareEqual(), combineScalarAndWithMaskSetcc(), combineX86ShufflesRecursively(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_AE, llvm::X86::COND_B, convertIntLogicToFPLogic(), llvm::ISD::DELETED_NODE, llvm::APInt::exactLogBase2(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FAND, llvm::APInt::getAllOnes(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), getBitWidth(), getBT(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNegative(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetConstantFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), I, Idx, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isBitwiseNot(), llvm::isConstOrConstSplat(), llvm::isOneConstant(), llvm::SDNode::isOnlyUserOf(), llvm::APInt::isPowerOf2(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), isZero(), LHS, llvm::SelectionDAG::MaskedValueIsZero(), matchScalarReduction(), llvm::X86::MaxShuffleCombineDepth, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, N, RHS, llvm::APInt::setBit(), llvm::ISD::SETEQ, llvm::ISD::SHL, llvm::TargetLowering::SimplifyDemandedBits(), llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::APInt::uge(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineAndLoadToBZHI()

static SDValue combineAndLoadToBZHI ( SDNode *  Node, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 50524 of file X86ISelLowering.cpp.

References llvm::AllOnes, llvm::ISD::AND, GEP, llvm::SelectionDAG::getAllOnesConstant(), llvm::Type::getArrayElementType(), llvm::Type::getArrayNumElements(), llvm::SelectionDAG::getConstant(), getIndexFromUnindexedLoad(), llvm::SelectionDAG::getNode(), llvm::Type::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getZExtOrTrunc(), hasBZHI(), llvm::Type::isIntegerTy(), llvm::ISD::SRL, llvm::ISD::SUB, and llvm::ISD::TRUNCATE.

Referenced by combineAnd().

combineAndMaskToShift()

static SDValue combineAndMaskToShift ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

If this is a zero/all-bits result that is bitwise-anded with a low bits mask.

(Mask == 1 for the x86 lowering of a SETCC + ZEXT), replace the 'and' with a shift-right to eliminate loading the vector constant mask value.

Definition at line 50406 of file X86ISelLowering.cpp.

References llvm::X86ISD::ANDNP, llvm::SelectionDAG::ComputeNumSignBits(), llvm::APInt::countr_one(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::isAllOnesOrAllOnesSplat(), llvm::isBitwiseNot(), llvm::X86::isConstantSplat(), llvm::EVT::isInteger(), llvm::APInt::isMask(), llvm::EVT::isSimple(), N, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), llvm::ISD::SRA, llvm::ISD::SRL, supportedVectorShiftWithImm(), llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, X, and Y.

Referenced by combineAnd().

combineAndNotIntoANDNP()

static SDValue combineAndNotIntoANDNP ( SDNode *  N, SelectionDAG &  DAG  )

static

Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).

Definition at line 50020 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), IsNOT(), N, X, and Y.

Referenced by combineAnd().

combineAndNotOrIntoAndNotAnd()

static SDValue combineAndNotOrIntoAndNotAnd ( SDNode *  N, SelectionDAG &  DAG  )

static

Folds (and X, (or Y, ~Z)) --> (and X, ~(and ~Y, Z)) This undoes the inverse fold performed in InstCombine.

Definition at line 50490 of file X86ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::hasAndNot(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::PatternMatch::m_And(), llvm::PatternMatch::m_Not(), llvm::PatternMatch::m_OneUse(), llvm::PatternMatch::m_Or(), llvm::PatternMatch::m_Value(), N, X, and Y.

Referenced by combineAnd().

combineAndnp()

static SDValue combineAndnp ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Do target-specific dag combines on X86ISD::ANDNP nodes.

Definition at line 54530 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::AND, llvm::ISD::BITCAST, llvm::APInt::clearAllBits(), combineX86ShufflesRecursively(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::DELETED_NODE, DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelect(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), I, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), IsNOT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::MVT::isVector(), isZero(), N, llvm::ISD::OR, llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::APInt::setBit(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::TargetLowering::SimplifyDemandedBits(), and llvm::TargetLowering::SimplifyDemandedVectorElts().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineAndOrForCcmpCtest()

static SDValue combineAndOrForCcmpCtest ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  ST  )

static

Definition at line 50805 of file X86ISelLowering.cpp.

References llvm::X86ISD::CCMP, llvm::X86ISD::CMP, llvm::X86::COND_NP, llvm::X86::COND_P, llvm::X86ISD::CTEST, DL, llvm::SDNode::getAsAPIntVal(), llvm::X86::getCCMPCondFlagsFromCondCode(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::APInt::getSExtValue(), llvm::SelectionDAG::getTargetConstant(), llvm::isNullConstant(), N, llvm::ISD::OR, llvm::X86ISD::SETCC, llvm::X86ISD::SUB, and std::swap().

Referenced by combineAnd(), and combineOr().

combineAndShuffleNot()

static SDValue combineAndShuffleNot ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Try to fold: and (vector_shuffle<Z,...,Z> (insert_vector_elt undef, (xor X, -1), Z), undef), Y -> andnp (vector_shuffle<Z,...,Z> (insert_vector_elt undef, X, Z), undef), Y.

Definition at line 50051 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), IsNOT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), N, llvm::peekThroughOneUseBitcasts(), splitVector(), llvm::X86Subtarget::useAVX512Regs(), X, and Y.

Referenced by combineAnd().

combineArithReduction()

static SDValue combineArithReduction ( SDNode *  ExtElt, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Try to convert a vector reduction sequence composed of binops and shuffles into horizontal ops.

Definition at line 46007 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::computeKnownBits(), llvm::ISD::CONCAT_VECTORS, DL, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::KnownBits::getMaxValue(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::ISD::INSERT_VECTOR_ELT, llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::Lo, llvm::Log2_32(), llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::MUL, llvm::X86ISD::PACKUS, llvm::X86ISD::PSADBW, shouldUseHorizontalOp(), SplitOpsAndApply(), splitVector(), llvm::ISD::TRUNCATE, llvm::APInt::ule(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineExtractVectorElt().

combineAVG()

static SDValue combineAVG ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 54227 of file X86ISelLowering.cpp.

References llvm::ISD::AVGCEILS, llvm::ISD::AVGCEILU, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::APInt::getSignMask(), llvm::EVT::isVector(), N, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBasicSADPattern()

static SDValue combineBasicSADPattern ( SDNode *  Extract, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 45581 of file X86ISelLowering.cpp.

References llvm::ISD::ABS, llvm::ISD::ADD, llvm::ISD::ANY_EXTEND, createPSADBW(), detectZextAbsDiff(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::isPowerOf2_32(), llvm::Log2_32(), llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by combineExtractVectorElt().

combineBEXTR()

static SDValue combineBEXTR ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 54251 of file X86ISelLowering.cpp.

References llvm::APInt::getAllOnes(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), N, and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBitcast()

static SDValue combineBitcast ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 44879 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, llvm::EVT::changeVectorElementTypeToInteger(), combineBitcastToBoolVector(), combineBitcastvxi1(), combineCastedMaskArithmetic(), combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::countl_zero(), createMMXBuildVector(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, F64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_TO_SINT, llvm::X86ISD::FXOR, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::APFloatBase::IEEEdouble(), llvm::EVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::isNullConstantOrUndef(), llvm::isPowerOf2_32(), llvm::EVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86ISD::MMX_MOVW2D, llvm::X86ISD::MOVDQ2Q, llvm::X86ISD::MOVMSK, N, llvm::SDNode::ops(), llvm::ISD::OR, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SETLT, llvm::ISD::TRUNCATE, llvm::X86ISD::VBROADCAST_LOAD, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBitcastToBoolVector()

static SDValue combineBitcastToBoolVector ( EVT  VT, SDValue  V, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, unsigned  Depth = 0  )

static

Definition at line 44790 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::CallingConv::C, combineBitcastToBoolVector(), llvm::ISD::Constant, llvm::Depth, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86ISD::KSHIFTL, llvm::SelectionDAG::MaxRecursionDepth, llvm::ISD::OR, llvm::ISD::SHL, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by combineBitcast(), and combineBitcastToBoolVector().

combineBitcastvxi1()

static SDValue combineBitcastvxi1 ( SelectionDAG &  DAG, EVT  VT, SDValue  Src, const SDLoc &  DL, const X86Subtarget &  Subtarget  )

static

Definition at line 44488 of file X86ISelLowering.cpp.

References adjustBitcastSrcVectorSSE1(), llvm::all_of(), llvm::ISD::ANY_EXTEND, llvm::SmallVectorTemplateCommon< T, typename >::begin(), checkBitcastSrcVectorSize(), collectConcatOps(), combineBitcastvxi1(), DL, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), getPMOVMSKB(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isSimple(), llvm::X86ISD::MOVMSK, llvm::ISD::SETCC, llvm::ISD::SETLT, llvm::ISD::SIGN_EXTEND, signExtendBitcastSrcVector(), llvm::MVT::SimpleTy, llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::TRUNCATE, and widenSubVector().

Referenced by combineAnd(), combineBitcast(), combineBitcastvxi1(), combineExtractVectorElt(), combineOr(), and combinePredicateReduction().

combineBitOpWithMOVMSK()

static SDValue combineBitOpWithMOVMSK ( unsigned  Opc, const SDLoc &  DL, SDValue  N0, SDValue  N1, SelectionDAG &  DAG  )

static

Definition at line 50299 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), convertIntLogicToFPLogicOpcode(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::isFloatingPoint(), llvm::X86ISD::MOVMSK, llvm::ISD::OR, and llvm::ISD::XOR.

Referenced by combineAnd(), combineOr(), and combineXor().

combineBitOpWithPACK()

static SDValue combineBitOpWithPACK ( unsigned  Opc, const SDLoc &  DL, EVT  VT, SDValue  N0, SDValue  N1, SelectionDAG &  DAG  )

static

Definition at line 50368 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::hasOneUse(), LHS, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::peekThroughOneUseBitcasts(), RHS, and llvm::ISD::XOR.

Referenced by combineAnd(), combineOr(), and combineXor().

combineBitOpWithShift()

static SDValue combineBitOpWithShift ( unsigned  Opc, const SDLoc &  DL, EVT  VT, SDValue  N0, SDValue  N1, SelectionDAG &  DAG  )

static

Definition at line 50330 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::ISD::OR, llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and llvm::ISD::XOR.

Referenced by combineAnd(), combineOr(), and combineXor().

combineBITREVERSE()

static SDValue combineBITREVERSE ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 54199 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSSE3(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBlendOfPermutes()

static SDValue combineBlendOfPermutes ( MVT  VT, SDValue  N0, SDValue  N1, ArrayRef< int >  BlendMask, const APInt &  DemandedElts, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, const SDLoc &  DL  )

static

Definition at line 41272 of file X86ISelLowering.cpp.

References assert(), canScaleShuffleElements(), canWidenShuffleElements(), DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::getShuffleDemandedElts(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), I, llvm::APInt::intersects(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), isBlendOrUndef(), isLaneCrossingShuffleMask(), isUndefOrEqual(), isUndefOrInRange(), llvm::APInt::isZero(), llvm::peekThroughOneUseBitcasts(), scaleShuffleElements(), and llvm::SM_SentinelUndef.

Referenced by llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

combineBMILogicOp()

static SDValue combineBMILogicOp ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 50724 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), getBMIMatchingOp(), llvm::EVT::isScalarInteger(), N, and llvm::ISD::XOR.

Referenced by combineAnd(), and combineXor().

combineBrCond()

static SDValue combineBrCond ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Optimize branch condition evaluation.

Definition at line 55893 of file X86ISelLowering.cpp.

References llvm::X86ISD::BRCOND, CC, combineSetCCEFLAGS(), Cond, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBROADCAST_LOAD()

static SDValue combineBROADCAST_LOAD ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 58869 of file X86ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::EVT::getFixedSizeInBits(), llvm::EVT::getSizeInBits(), N, Ptr, llvm::X86ISD::SUBV_BROADCAST_LOAD, and llvm::X86ISD::VBROADCAST_LOAD.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineBT()

static SDValue combineBT ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 54675 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::BitWidth, llvm::ISD::DELETED_NODE, llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::Log2_32(), N, and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineCarryThroughADD()

static SDValue combineCarryThroughADD ( SDValue  EFLAGS, SelectionDAG &  DAG  )

static

Definition at line 47799 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADD, llvm::ISD::AND, llvm::X86::COND_A, llvm::X86::COND_B, llvm::X86::COND_E, DL, getBT(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::SDValue::getValueType(), llvm::SDNode::getVTList(), llvm::SDNode::hasOneUse(), llvm::isAllOnesConstant(), llvm::EVT::isInteger(), llvm::isOneConstant(), llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::ISD::SRL, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

combineCastedMaskArithmetic()

static SDValue combineCastedMaskArithmetic ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 44654 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, combinevXi1ConstantToInteger(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX512(), llvm::Value::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isScalarInteger(), llvm::EVT::isVector(), LHS, N, llvm::ISD::OR, RHS, and llvm::ISD::XOR.

Referenced by combineBitcast().

combineCMov()

static SDValue combineCMov ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].

Definition at line 48280 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADC, llvm::Add, llvm::ISD::ADD, assert(), llvm::X86Subtarget::canUseCMOV(), CC, checkBoolTestAndOrSetCCCombine(), llvm::X86ISD::CMOV, llvm::X86ISD::CMP, combineSetCCEFLAGS(), Cond, llvm::X86::COND_AE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::ISD::CTTZ, llvm::ISD::CTTZ_ZERO_UNDEF, DL, llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::APInt::getZExtValue(), hasFPCMov(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::ISD::MUL, N, llvm::ISD::SHL, llvm::ISD::SUB, llvm::X86ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, llvm::APInt::ult(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineCMP()

static SDValue combineCMP ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 56249 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::ISD::AND, llvm::X86ISD::AND, llvm::BitWidth, llvm::X86ISD::CMP, combineX86SubCmpForFlags(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::APInt::getZExtValue(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86ISD::KSHIFTR, llvm::SelectionDAG::MaskedValueIsZero(), N, needCarryOrOverflowFlag(), onlyZeroFlagUsed(), llvm::ISD::OR, llvm::X86ISD::OR, llvm::ISD::SHL, llvm::ISD::SRL, llvm::ISD::SUB, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, llvm::APInt::ult(), widenMaskVector(), llvm::ISD::XOR, llvm::X86ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineCommutableSHUFP()

static SDValue combineCommutableSHUFP ( SDValue  N, MVT  VT, const SDLoc &  DL, SelectionDAG &  DAG  )

static

Definition at line 41217 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::X86::mayFoldLoad(), N, llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::SHUFP, and llvm::X86ISD::VPERMILPI.

Referenced by combineTargetShuffle().

combineCompareEqual()

static SDValue combineCompareEqual ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends.

Likewise for OR -> CMPNEQSS.

Definition at line 49913 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BR_CC, llvm::ISD::BRCOND, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_P, llvm::ISD::CopyToReg, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FCMP, llvm::X86ISD::FSETCC, llvm::X86ISD::FSETCCM, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, isAndOrOfSetCCs(), N, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SELECT, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineAnd(), and combineOr().

combineCONCAT_VECTORS()

static SDValue combineCONCAT_VECTORS ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 57893 of file X86ISelLowering.cpp.

References llvm::CallingConv::C, combineConcatVectorOps(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX(), I, llvm::TargetLoweringBase::isTypeLegal(), N, llvm::peekThroughBitcasts(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineConcatVectorOps()

static SDValue combineConcatVectorOps ( const SDLoc &  DL, MVT  VT, ArrayRef< SDValue >  Ops, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.

ISD::INSERT_SUBVECTOR). The ops are assumed to be of the same type.

Definition at line 57181 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::all_equal(), llvm::all_of(), llvm::X86TargetLowering::allowsMemoryAccess(), llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::ISD::ANY_EXTEND, llvm::SmallVectorImpl< T >::append(), assert(), llvm::X86ISD::BLENDI, llvm::X86ISD::BLENDV, llvm::CallingConv::C, CastIntSETCCtoFP(), llvm::EVT::changeVectorElementType(), llvm::X86ISD::CMPP, llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::ISD::CONCAT_VECTORS, concatSubVectors(), llvm::ISD::CTLZ, llvm::ISD::CTLZ_ZERO_UNDEF, llvm::ISD::CTPOP, llvm::ISD::CTTZ, llvm::ISD::CTTZ_ZERO_UNDEF, DL, EltsFromConsecutiveLoads(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::ISD::FADD, llvm::CallingConv::Fast, llvm::ISD::FDIV, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), getBROADCAST_LOAD(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), getConstantVector(), getConstVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getEntryNode(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::X86Subtarget::getPreferVectorWidth(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), getTargetConstantFromNode(), llvm::X86Subtarget::getTargetLowering(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), getZeroVector(), llvm::X86ISD::GF2P8AFFINEQB, llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86ISD::HSUB, I, Idx, llvm::APInt::insertBits(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::MVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isFloatingPoint(), llvm::MVT::isFloatingPoint(), llvm::ISD::isNormalLoad(), llvm::EVT::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LHS, llvm::X86::mayFoldLoad(), llvm::X86::mayFoldLoadIntoBroadcastFromMem(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::ISD::MUL, llvm::none_of(), llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), llvm::X86ISD::PSADBW, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SETEQ, llvm::ISD::SETGT, llvm::ISD::SETOEQ, llvm::ISD::SETOGT, llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::ISD::SIGN_EXTEND, llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::SUB, llvm::X86ISD::SUBV_BROADCAST_LOAD, translateX86FSETCC(), llvm::ISD::TRUNCATE, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86Subtarget::useAVX512Regs(), llvm::X86Subtarget::useBWIRegs(), llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::ISD::VECTOR_SHUFFLE, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPMADDUBSW, llvm::X86ISD::VPMADDWD, llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::ISD::VSELECT, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, llvm::X86ISD::VSRLI, widenSubVector(), llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by combineCONCAT_VECTORS(), and combineINSERT_SUBVECTOR().

combineConstantPoolLoads()

static SDValue combineConstantPoolLoads ( SDNode *  N, const SDLoc &  dl, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 52148 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::EVT::getFixedSizeInBits(), llvm::Type::getPrimitiveSizeInBits(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetConstantFromBasePtr(), llvm::Value::getType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX(), I, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::ISD::isNormalLoad(), N, llvm::ISD::NON_EXTLOAD, Ptr, llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::SDNode::users(), and llvm::X86ISD::VBROADCAST_LOAD.

Referenced by combineLoad().

combineCVTP2I_CVTTP2I()

static SDValue combineCVTP2I_CVTTP2I ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 54492 of file X86ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getFloatingPointVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelectionDAGInfo(), llvm::SDValue::getValue(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::ISD::isNormalLoad(), llvm::SelectionDAGTargetInfo::isTargetStrictFPOpcode(), N, narrowLoadToVZLoad(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), and llvm::SelectionDAG::ReplaceAllUsesOfValueWith().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineCVTPH2PS()

static SDValue combineCVTPH2PS ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 54691 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::DELETED_NODE, llvm::SelectionDAG::getBitcast(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::ISD::isNormalLoad(), N, narrowLoadToVZLoad(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::TargetLowering::SimplifyDemandedVectorElts(), and llvm::X86ISD::STRICT_CVTPH2PS.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineEXTEND_VECTOR_INREG()

static SDValue combineEXTEND_VECTOR_INREG ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 58645 of file X86ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::EVT::changeVectorElementType(), combineX86ShufflesRecursively(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getExtLoad(), llvm::SDNode::getFlags(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getOpcode_EXTEND(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE41(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isLoadExtLegal(), llvm::ISD::isNormalLoad(), llvm::TargetLoweringBase::isTypeLegal(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::ZERO_EXTEND_VECTOR_INREG, and llvm::ISD::ZEXTLOAD.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineEXTRACT_SUBVECTOR()

static SDValue combineEXTRACT_SUBVECTOR ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 58140 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::BLENDI, llvm::ISD::BUILD_VECTOR, llvm::X86ISD::CMPP, Concat, llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTUI2P, DL, llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::ISD::FP_EXTEND, llvm::ISD::FP_TO_SINT, llvm::ISD::FP_TO_UINT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isBitwiseNot(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isExtOpcode(), llvm::ISD::isExtVecInRegOpcode(), llvm::SelectionDAG::isSplatValue(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::LOAD, llvm::X86ISD::MOVDDUP, N, narrowExtractedVectorSelect(), llvm::SDNode::ops(), llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFD, scaleShuffleElements(), llvm::ISD::SINT_TO_FP, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, splitVectorIntBinary(), llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::ISD::TRUNCATE, llvm::ISD::UINT_TO_FP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VFPEXT, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMV3, llvm::ISD::VSELECT, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLI, and widenSubVector().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineExtractFromVectorLoad()

static SDValue combineExtractFromVectorLoad ( SDNode *  N, EVT  VecVT, SDValue  SrcVec, uint64_t  Idx, const SDLoc &  dl, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 45664 of file X86ISelLowering.cpp.

References llvm::any_of(), assert(), llvm::commonAlignment(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getLoad(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::TargetLowering::getVectorElementPointer(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MachinePointerInfo::getWithOffset(), Idx, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), and N.

Referenced by combineExtractVectorElt(), and combineExtractWithShuffle().

combineExtractVectorElt()

static SDValue combineExtractVectorElt ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.

For i686, the best sequence is apparently storing the value and loading scalars back, while for x64 we should use 64-bit extracts and shifts.

Definition at line 46200 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, assert(), llvm::ISD::BITCAST, combineArithReduction(), combineBasicSADPattern(), combineBitcastvxi1(), combineExtractFromVectorLoad(), combineExtractWithShuffle(), combineMinMaxReduction(), combinePredicateReduction(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineVPDPBUSDPattern(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), Idx, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86ISD::MMX_MOVD2W, N, llvm::peekThroughBitcasts(), llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), scalarizeExtEltFP(), llvm::ISD::SETEQ, llvm::ISD::SHL, llvm::TargetLowering::SimplifyDemandedBits(), llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::TRUNCATE, and llvm::SDNode::users().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineExtractWithShuffle()

static SDValue combineExtractWithShuffle ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 45701 of file X86ISelLowering.cpp.

References llvm::any_of(), assert(), canWidenShuffleElements(), combineExtractFromVectorLoad(), extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::APInt::getZExtValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, Idx, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::EVT::isScalarInteger(), isUndefOrZeroInRange(), llvm::Lo, N, llvm::narrowShuffleMaskElts(), llvm::Offset, llvm::peekThroughBitcasts(), llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::APInt::uge(), llvm::APInt::ult(), llvm::APInt::urem(), llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VBROADCAST_LOAD.

Referenced by combineExtractVectorElt().

combineExtSetcc()

static SDValue combineExtSetcc ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 54962 of file X86ISelLowering.cpp.

References CC, llvm::EVT::changeVectorElementTypeToInteger(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86Subtarget::hasAVX512(), llvm::ISD::isUnsignedIntSetCC(), llvm::EVT::isVector(), N, llvm::ISD::SETCC, Size, llvm::X86Subtarget::useAVX512Regs(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineSext(), and combineZext().

combineFaddCFmul()

static SDValue combineFaddCFmul ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 53282 of file X86ISelLowering.cpp.

References llvm::TargetOptions::AllowFPOpFusion, llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), llvm::ISD::FADD, llvm::FPOpFusion::Fast, llvm::SelectionDAG::getBitcast(), llvm::SDNode::getFlags(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTarget(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllZeros(), LHS, N, llvm::TargetOptions::NoSignedZerosFPMath, llvm::TargetMachine::Options, RHS, llvm::X86ISD::VFCMADDC, llvm::X86ISD::VFCMULC, llvm::X86ISD::VFMADDC, and llvm::X86ISD::VFMULC.

Referenced by combineFaddFsub().

combineFaddFsub()

static SDValue combineFaddFsub ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Do target-specific dag combines on floating-point adds/subs.

Definition at line 53357 of file X86ISelLowering.cpp.

References combineFaddCFmul(), combineToHorizontalAddSub(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFAnd()

static SDValue combineFAnd ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Do target-specific dag combines on X86ISD::FAND nodes.

Definition at line 54321 of file X86ISelLowering.cpp.

References combineFAndFNotToFAndn(), getNullFPConstForNullVal(), lowerX86FPLogicOp(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFAndFNotToFAndn()

static SDValue combineFAndFNotToFAndn ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 54289 of file X86ISelLowering.cpp.

References llvm::CallingConv::C, DL, llvm::X86ISD::FANDN, llvm::X86ISD::FXOR, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), and N.

Referenced by combineFAnd().

combineFAndn()

static SDValue combineFAndn ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Do target-specific dag combines on X86ISD::FANDN nodes.

Definition at line 54338 of file X86ISelLowering.cpp.

References getNullFPConstForNullVal(), isNullFPScalarOrVectorConst(), lowerX86FPLogicOp(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFMA()

static SDValue combineFMA ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 55111 of file X86ISelLowering.cpp.

References A, assert(), B, llvm::CallingConv::C, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FMA, llvm::ISD::FMUL, llvm::TargetLowering::getCheaperNegatedExpression(), llvm::MachineFunction::getFunction(), getInvertedVectorForFMA(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelectionDAGInfo(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasAnyFMA(), llvm::Function::hasOptSize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::isNullConstant(), llvm::TargetLoweringBase::isOperationExpand(), llvm::SelectionDAGTargetInfo::isTargetStrictFPOpcode(), llvm::TargetLoweringBase::isTypeLegal(), N, and negateFMAOpcode().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFMADDSUB()

static SDValue combineFMADDSUB ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 55203 of file X86ISelLowering.cpp.

References llvm::TargetLowering::getCheaperNegatedExpression(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasOptSize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, and negateFMAOpcode().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFMinFMax()

static SDValue combineFMinFMax ( SDNode *  N, SelectionDAG &  DAG  )

static

Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.

Definition at line 54372 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::FMAX, llvm::X86ISD::FMAXC, llvm::X86ISD::FMIN, llvm::X86ISD::FMINC, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTarget(), llvm_unreachable, N, llvm::TargetOptions::NoNaNsFPMath, llvm::TargetOptions::NoSignedZerosFPMath, and llvm::TargetMachine::Options.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFMinNumFMaxNum()

static SDValue combineFMinNumFMaxNum ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 54393 of file X86ISelLowering.cpp.

References DL, llvm::X86ISD::FMAX, llvm::ISD::FMAXNUM, llvm::X86ISD::FMIN, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasMinSize(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::SelectionDAG::isKnownNeverNaN(), isSoftF16(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, llvm::TargetOptions::NoNaNsFPMath, llvm::TargetMachine::Options, and llvm::ISD::SETUO.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFMulcFCMulc()

static SDValue combineFMulcFCMulc ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 53241 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::getBitcast(), llvm::KnownBits::getBitWidth(), llvm::KnownBits::getConstant(), llvm::SelectionDAG::getNode(), llvm::Value::hasOneUse(), llvm::KnownBits::isConstant(), LHS, N, RHS, std::swap(), llvm::X86ISD::VFCMULC, llvm::X86ISD::VFMULC, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFneg()

static SDValue combineFneg ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Do target-specific dag combines on floating point negations.

Definition at line 53903 of file X86ISelLowering.cpp.

References DL, llvm::ISD::FMUL, llvm::X86ISD::FNMSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantFP(), llvm::SDNode::getFlags(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::TargetLowering::getNegatedExpression(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDNodeFlags::hasNoSignedZeros(), llvm::Function::hasOptSize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isFNEG(), llvm::TargetLoweringBase::isTypeLegal(), and N.

Referenced by combineFOr(), combineXor(), and llvm::X86TargetLowering::PerformDAGCombine().

combineFOr()

static SDValue combineFOr ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.

Definition at line 54352 of file X86ISelLowering.cpp.

References assert(), combineFneg(), llvm::X86ISD::FOR, llvm::X86ISD::FXOR, isNullFPScalarOrVectorConst(), lowerX86FPLogicOp(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFP16_TO_FP()

static SDValue combineFP16_TO_FP ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 58750 of file X86ISelLowering.cpp.

References llvm::X86ISD::CVTPH2PS, llvm::X86ISD::CVTPS2PH, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP_TO_FP16, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), N, and llvm::ISD::SCALAR_TO_VECTOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFP_EXTEND()

static SDValue combineFP_EXTEND ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 58772 of file X86ISelLowering.cpp.

References assert(), llvm::EVT::changeTypeToInteger(), llvm::EVT::changeVectorElementType(), llvm::EVT::changeVectorElementTypeToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTPH2PS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), N, llvm::ISD::SHL, llvm::X86ISD::STRICT_CVTPH2PS, llvm::ISD::STRICT_FP_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineFP_ROUND()

static SDValue combineFP_ROUND ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 58905 of file X86ISelLowering.cpp.

References assert(), llvm::EVT::changeVectorElementTypeToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTPS2PH, llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTUI2P, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::isPowerOf2_32(), llvm::SDNode::isStrictFPOpcode(), llvm::EVT::isVector(), N, llvm::ISD::SINT_TO_FP, llvm::X86ISD::STRICT_CVTPS2PH, llvm::X86ISD::STRICT_CVTSI2P, llvm::X86ISD::STRICT_CVTUI2P, and llvm::ISD::STRICT_SINT_TO_FP.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineGatherScatter()

static SDValue combineGatherScatter ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 55769 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::sampleprof::Base, llvm::CallingConv::C, llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::DELETED_NODE, DL, llvm::SDNode::getAsZExtVal(), llvm::SelectionDAG::getConstant(), llvm::BuildVectorSDNode::getConstantSplatNode(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getSplatBuildVector(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::BuildVectorSDNode::isConstant(), llvm::isOneConstant(), N, llvm::BitVector::none(), rebuildGatherScatter(), llvm::ISD::SIGN_EXTEND, llvm::TargetLowering::SimplifyDemandedBits(), llvm::Splat, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineHorizOpWithShuffle()

static SDValue combineHorizOpWithShuffle ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 49345 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::SmallVectorBase< Size_T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getSplitVectorSrc(), getTargetShuffleInputs(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasInt256(), llvm::Hi, Idx, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), isAnyZero(), llvm::EVT::isFloatingPoint(), isHorizOp(), LHS, llvm::Lo, N, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), resolveTargetShuffleInputsAndMask(), RHS, scaleShuffleElements(), llvm::SelectionDAG::SplitVector(), and std::swap().

Referenced by combineVectorHADDSUB(), and combineVectorPack().

combineINSERT_SUBVECTOR()

static SDValue combineINSERT_SUBVECTOR ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 57929 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), collectConcatOps(), combineConcatVectorOps(), combineX86ShufflesRecursively(), llvm::ISD::EXTRACT_SUBVECTOR, getBROADCAST_LOAD(), llvm::SDValue::getConstantOperandVal(), llvm::MVT::getFixedSizeInBits(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllZeros(), llvm::isNullConstant(), isTargetShuffle(), llvm::SDValue::isUndef(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VBROADCAST_LOAD.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineINTRINSIC_VOID()

static SDValue combineINTRINSIC_VOID ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 59093 of file X86ISelLowering.cpp.

References FixupMMXIntrinsicTypes(), llvm::getIntrinsicWithChain(), llvm::INTR_TYPE_CAST_MMX, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, and llvm::IntrinsicData::Type.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineINTRINSIC_W_CHAIN()

static SDValue combineINTRINSIC_W_CHAIN ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 59079 of file X86ISelLowering.cpp.

References FixupMMXIntrinsicTypes(), llvm::getIntrinsicWithChain(), llvm::INTR_TYPE_CAST_MMX, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, and llvm::IntrinsicData::Type.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineINTRINSIC_WO_CHAIN()

static SDValue combineINTRINSIC_WO_CHAIN ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 59065 of file X86ISelLowering.cpp.

References FixupMMXIntrinsicTypes(), llvm::getIntrinsicWithoutChain(), llvm::INTR_TYPE_CAST_MMX, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, and llvm::IntrinsicData::Type.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineKSHIFT()

static SDValue combineKSHIFT ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 58713 of file X86ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorAllZeros(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86ISD::KSHIFTR, N, and llvm::TargetLowering::SimplifyDemandedVectorElts().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineLoad()

static SDValue combineLoad ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 52223 of file X86ISelLowering.cpp.

References llvm::TargetLoweringBase::allowsMemoryAccess(), combineConstantPoolLoads(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, extractSubVector(), llvm::CallingConv::Fast, llvm::SelectionDAG::getAddrSpaceCast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getExtLoad(), llvm::TypeSize::getFixed(), llvm::EVT::getFixedSizeInBits(), llvm::SDNode::getFlags(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarType(), llvm::SDNode::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, llvm::ISD::NON_EXTLOAD, Ptr, llvm::X86AS::PTR32_SPTR, llvm::X86AS::PTR32_UPTR, llvm::X86AS::PTR64, llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::ISD::TokenFactor, and llvm::SDNode::users().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineLogicBlendIntoConditionalNegate()

static SDValue combineLogicBlendIntoConditionalNegate ( EVT  VT, SDValue  Mask, SDValue  X, SDValue  Y, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 46793 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isOperationLegal(), N, llvm::ISD::SUB, std::swap(), X, llvm::ISD::XOR, and Y.

Referenced by combineLogicBlendIntoPBLENDV(), and combineSelect().

combineLogicBlendIntoPBLENDV()

static SDValue combineLogicBlendIntoPBLENDV ( SDNode *  N, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 51258 of file X86ISelLowering.cpp.

References assert(), combineLogicBlendIntoConditionalNegate(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::isInteger(), matchLogicBlend(), N, llvm::ISD::OR, llvm::peekThroughBitcasts(), X, and Y.

Referenced by combineOr().

combineLRINT_LLRINT()

static SDValue combineLRINT_LLRINT ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 53368 of file X86ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTP2SI, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMaskedLoad()

static SDValue combineMaskedLoad ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 52485 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), combineMaskedLoadConstantMask(), llvm::ISD::DELETED_NODE, llvm::SelectionDAG::getMaskedLoad(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasAVX512(), N, llvm::ISD::NON_EXTLOAD, reduceMaskedLoadToScalarLoad(), llvm::TargetLowering::SimplifyDemandedBits(), and llvm::TargetLowering::SimplifyMultipleUseDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMaskedLoadConstantMask()

static SDValue combineMaskedLoadConstantMask ( MaskedLoadSDNode *  ML, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 52437 of file X86ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMaskedLoad(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isNullConstant(), and ML.

Referenced by combineMaskedLoad().

combineMaskedStore()

static SDValue combineMaskedStore ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 52564 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::DELETED_NODE, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::MaskedStoreSDNode::getOffset(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MaskedStoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::Value::hasOneUse(), llvm::MaskedStoreSDNode::isCompressingStore(), llvm::MaskedStoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegal(), N, reduceMaskedStoreToScalarStore(), llvm::TargetLowering::SimplifyDemandedBits(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), and llvm::ISD::TRUNCATE.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMinMaxReduction()

static SDValue combineMinMaxReduction ( SDNode *  Extract, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 45297 of file X86ISelLowering.cpp.

References assert(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, llvm::Lo, llvm::SelectionDAG::matchBinOpReduction(), llvm::X86ISD::PHMINPOS, llvm::ISD::SMAX, llvm::ISD::SMIN, splitVector(), llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::Upper, and llvm::ISD::XOR.

Referenced by combineExtractVectorElt().

combineMOVDQ2Q()

static SDValue combineMOVDQ2Q ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 58996 of file X86ISelLowering.cpp.

References llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::SDValue::getValue(), llvm::ISD::isNormalLoad(), llvm::MemSDNode::isSimple(), N, and llvm::SelectionDAG::ReplaceAllUsesOfValueWith().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMOVMSK()

static SDValue combineMOVMSK ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 55594 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMaxPopulation(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftByConstNode(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), llvm::X86Subtarget::hasSSE2(), Idx, llvm::ISD::isBitwiseLogicOp(), llvm::ISD::isBuildVectorAllOnes(), IsNOT(), llvm::KnownBits::isZero(), llvm::X86ISD::MOVMSK, N, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughOneUseBitcasts(), llvm::TargetLowering::SimplifyDemandedBits(), llvm::X86ISD::VSHLI, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMul()

static SDValue combineMul ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 48905 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::CallingConv::C, combineMulSpecial(), combineMulToPMADDWD(), combineMulToPMULDQ(), llvm::SelectionDAG::computeKnownBits(), DL, llvm::KnownBits::getConstant(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::Function::hasMinSize(), INT64_MIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm::KnownBits::isConstant(), llvm::EVT::isInteger(), llvm::isPowerOf2_64(), llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::Log2_64(), llvm::X86ISD::MUL_IMM, MulConstantOptimization, N, reduceVMULWidth(), llvm::ISD::SHL, llvm::ISD::SUB, and std::swap().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineMulSpecial()

static SDValue combineMulSpecial ( uint64_t  MulAmt, SDNode *  N, SelectionDAG &  DAG, EVT  VT, const SDLoc &  DL  )

static

Definition at line 48663 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::countr_zero(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isPowerOf2_64(), llvm::Log2_64(), llvm::X86ISD::MUL_IMM, N, llvm::ISD::SHL, and llvm::ISD::SUB.

Referenced by combineMul().

combineMulToPMADDWD()

static SDValue combineMulToPMADDWD ( SDNode *  N, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 48752 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::ComputeMaxSignificantBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, SplitOpsAndApply(), llvm::X86ISD::VPMADDWD, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by combineMul().

combineMulToPMULDQ()

static SDValue combineMulToPMULDQ ( SDNode *  N, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 48863 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::X86ISD::PMULDQ, llvm::X86ISD::PMULUDQ, and SplitOpsAndApply().

Referenced by combineMul().

combineOr()

static SDValue combineOr ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 51705 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::AND, assert(), canonicalizeBitSelect(), combineAndOrForCcmpCtest(), combineBitcastvxi1(), combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineCompareEqual(), combineLogicBlendIntoPBLENDV(), combineOrXorWithSETCC(), combineX86ShufflesRecursively(), llvm::ISD::CONCAT_VECTORS, Cond, convertIntLogicToFPLogic(), llvm::ISD::DELETED_NODE, extractSubVector(), foldMaskedMerge(), llvm::X86ISD::FOR, llvm::SelectionDAG::getBitcast(), getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::EVT::getScalarSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), llvm::EVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86ISD::KSHIFTL, llvm::SelectionDAG::MaskedVectorIsZero(), matchScalarReduction(), llvm::ISD::MUL, N, llvm::ISD::OR, llvm::APInt::setBit(), llvm::X86ISD::SETCC, llvm::ISD::SETNE, llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::SUB, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineOrCmpEqZeroToCtlzSrl()

static SDValue combineOrCmpEqZeroToCtlzSrl ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 51333 of file X86ISelLowering.cpp.

References llvm::X86ISD::CMP, llvm::X86::COND_E, llvm::SmallVectorBase< Size_T >::empty(), llvm::SelectionDAG::getNode(), llvm::X86Subtarget::getTargetLowering(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::X86TargetLowering::isCtlzFast(), llvm::isNullConstant(), LHS, lowerX86CmpEqZeroToCtlzSrl(), N, llvm::ISD::OR, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::X86ISD::SETCC, std::swap(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineZext().

combineOrXorWithSETCC()

static SDValue combineOrXorWithSETCC ( unsigned  Opc, const SDLoc &  DL, EVT  VT, SDValue  N0, SDValue  N1, SelectionDAG &  DAG  )

static

Definition at line 51669 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), combineAddOrSubToADCOrSBB(), DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), isConstantPowerOf2(), llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::SETCC, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by combineOr(), and combineXor().

combinePDEP()

static SDValue combinePDEP ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 59017 of file X86ISelLowering.cpp.

References llvm::APInt::getAllOnes(), llvm::SelectionDAG::getTargetLoweringInfo(), N, and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combinePMULDQ()

static SDValue combinePMULDQ ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 58547 of file X86ISelLowering.cpp.

References llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVectorShuffle(), llvm::Value::hasOneUse(), llvm::ISD::isBuildVectorAllZeros(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), LHS, N, RHS, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::TargetLowering::SimplifyDemandedBits(), and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combinePMULH()

static SDValue combinePMULH ( SDValue  Src, EVT  VT, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 53470 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::KnownBits::countMaxActiveBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::EVT::is256BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isVector(), LHS, llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Value(), llvm::ISD::MULHS, llvm::ISD::MULHU, RHS, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineTruncate().

combinePredicateReduction()

static SDValue combinePredicateReduction ( SDNode *  Extract, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 45371 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::BitWidth, CC, combineBitcastvxi1(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MVT::getFloatingPointVT(), llvm::SelectionDAG::getFreeze(), llvm::EVT::getIntegerVT(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), getPMOVMSKB(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::Hi, llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), LHS, llvm::Lo, LowerVectorAllEqual(), llvm::Match, llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::OR, llvm::ISD::PARITY, RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::SelectionDAG::SplitVector(), llvm::ISD::TRUNCATE, and llvm::ISD::XOR.

Referenced by combineExtractVectorElt().

combinePTESTCC()

static SDValue combinePTESTCC ( SDValue  EFLAGS, X86::CondCode &  CC, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

If we are inverting an PTEST/TESTP operand, attempt to adjust the CC to avoid the inversion.

Definition at line 47858 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), CC, llvm::X86ISD::CMP, llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_INVALID, llvm::X86::COND_NE, DL, llvm::X86ISD::FAND, llvm::X86ISD::FANDN, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getDoubleNumVectorElementsVT(), llvm::MVT::getFloatingPointVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPMOVMSKB(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getSplitVectorSrc(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::EVT::is128BitVector(), llvm::MVT::is128BitVector(), llvm::ISD::isBuildVectorAllOnes(), IsNOT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::ISD::OR, llvm::peekThroughBitcasts(), llvm::X86ISD::PTEST, llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), and llvm::X86ISD::TESTP.

Referenced by combineSetCCEFLAGS().

combineRedundantDWordShuffle()

static SDValue combineRedundantDWordShuffle ( SDValue  N, MutableArrayRef< int >  Mask, const SDLoc &  DL, SelectionDAG &  DAG  )

static

Search for a combinable shuffle across a chain ending in pshufd.

We walk up the chain and look for a combinable shuffle, skipping over shuffles that we could hoist this shuffle's transformation past without altering anything.

Definition at line 41087 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, DL, llvm::SmallVectorBase< Size_T >::empty(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getPSHUFShuffleMask(), getV4X86ShuffleImm8ForMask(), llvm_unreachable, N, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by combineTargetShuffle().

combineSBB()

static SDValue combineSBB ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 56444 of file X86ISelLowering.cpp.

References combineCarryThroughADD(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::isNullConstant(), LHS, N, RHS, llvm::X86ISD::SBB, and llvm::ISD::SUB.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineScalarAndWithMaskSetcc()

static SDValue combineScalarAndWithMaskSetcc ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 50599 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BITCAST, Concat, llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::EVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, and llvm::ISD::SETCC.

Referenced by combineAnd().

combineScalarToVector()

static SDValue combineScalarToVector ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 58420 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::ISD::EXTLOAD, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::User::getOperand(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::KnownBits::isConstant(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::X86ISD::MOVQ2DQ, N, llvm::peekThroughOneUseBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, supportedVectorShiftWithImm(), llvm::X86ISD::VBROADCAST, llvm::X86ISD::VZEXT_MOVL, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSelect()

static SDValue combineSelect ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Do target-specific dag combines on SELECT and VSELECT nodes.

Definition at line 46870 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::BITCAST, llvm::X86ISD::BLENDV, llvm::CallingConv::C, CC, llvm::EVT::changeVectorElementType(), llvm::EVT::changeVectorElementTypeToInteger(), combineLogicBlendIntoConditionalNegate(), combineSelectOfTwoConstants(), combineToExtendBoolVectorInReg(), combineVSelectToBLENDV(), combineVSelectWithAllOnesOrZeros(), combinevXi1ConstantToInteger(), commuteSelect(), llvm::SelectionDAG::ComputeNumSignBits(), Cond, createShuffleMaskFromVSELECT(), DL, llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::X86ISD::FMAX, llvm::X86ISD::FMIN, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::Value::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, insertSubVector(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), isConstantPowerOf2(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::isConstOrConstSplat(), llvm::SelectionDAG::isEqualTo(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::SelectionDAG::isKnownNeverZeroFloat(), IsNOT(), llvm::isNullConstant(), llvm::isNullOrNullSplat(), llvm::isOneConstant(), isSoftF16(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), isUndefOrZero(), llvm::EVT::isVector(), LHS, llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), llvm::ISD::matchUnaryPredicate(), N, narrowVectorSelect(), llvm::TargetOptions::NoSignedZerosFPMath, llvm::TargetMachine::Options, llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::SELECT, Select, llvm::ISD::SETCC, llvm::ISD::SETCC_INVALID, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::simplifySelect(), llvm::ISD::SRL, llvm::X86ISD::STRICT_FMAX, llvm::X86ISD::STRICT_FMIN, llvm::ISD::STRICT_FSETCCS, supportedVectorVarShift(), std::swap(), llvm::ISD::TRUNCATE, llvm::X86ISD::VPERMV3, llvm::ISD::VSELECT, llvm::X86ISD::VSHLV, and llvm::X86ISD::VSRLV.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSelectOfTwoConstants()

static SDValue combineSelectOfTwoConstants ( SDNode *  N, SelectionDAG &  DAG, const SDLoc &  DL  )

static

Definition at line 46609 of file X86ISelLowering.cpp.

References llvm::APInt::abs(), llvm::ISD::ADD, CC, Cond, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::isNullConstant(), llvm::APInt::isOne(), llvm::APInt::isPowerOf2(), llvm::TargetLoweringBase::isTypeLegal(), LHS, llvm::ISD::MUL, N, RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, std::swap(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineSelect().

combineSetCC()

static SDValue combineSetCC ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 55305 of file X86ISelLowering.cpp.

References llvm::ISD::ABS, llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, CC, combineVectorSizedSetCCEquality(), llvm::SelectionDAG::computeKnownBits(), DL, llvm::ISD::FNEG, llvm::SelectionDAG::getAllOnesConstant(), llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNegative(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCSwappedOperands(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasOneUse(), llvm::Value::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), incDecVectorConstant(), llvm::KnownBits::intersectWith(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllZeros(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::EVT::isInteger(), llvm::APInt::isMinSignedValue(), isNullFPScalarOrVectorConst(), llvm::TargetLoweringBase::isOperationLegal(), llvm::APInt::isPowerOf2(), llvm::EVT::isScalarInteger(), llvm::APInt::isSignBitSet(), llvm::ISD::isSignedIntSetCC(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::isUnsignedIntSetCC(), llvm::EVT::isVector(), LHS, LowerVSETCC(), llvm::SelectionDAG::MaskedValueIsZero(), MatchVectorAllEqualTest(), N, llvm::KnownBits::One, llvm::ISD::OR, RHS, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, truncateAVX512SetCCNoBWI(), llvm::ISD::UMIN, and llvm::KnownBits::Zero.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSetCCAtomicArith()

static SDValue combineSetCCAtomicArith ( SDValue  Cmp, X86::CondCode &  CC, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction.

Note that this is only legal for some op/cc combinations.

Definition at line 47452 of file X86ISelLowering.cpp.

References llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_SUB, CC, llvm::X86ISD::CMP, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NS, llvm::X86::COND_S, llvm::SelectionDAG::getAtomic(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), lowerAtomicArithWithLOCK(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and llvm::X86ISD::SUB.

Referenced by combineSetCCEFLAGS().

combineSetCCEFLAGS()

static SDValue combineSetCCEFLAGS ( SDValue  EFLAGS, X86::CondCode &  CC, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values.

Definition at line 48257 of file X86ISelLowering.cpp.

References CC, checkBoolTestSetCCCombine(), checkSignTestSetCCCombine(), combineCarryThroughADD(), combinePTESTCC(), combineSetCCAtomicArith(), combineSetCCMOVMSK(), and llvm::X86::COND_B.

Referenced by combineBrCond(), combineCMov(), and combineX86SetCC().

combineSetCCMOVMSK()

static SDValue combineSetCCMOVMSK ( SDValue  EFLAGS, X86::CondCode &  CC, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 48022 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, canScaleShuffleElements(), CC, llvm::MVT::changeVectorElementTypeToInteger(), llvm::X86ISD::CMP, collectConcatOps(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getSplitVectorSrc(), getTargetShuffleInputs(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), isCompletePermute(), llvm::APInt::isMask(), llvm::SDValue::isUndef(), llvm::APInt::isZero(), LHS, llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PCMPEQ, llvm::peekThroughBitcasts(), llvm::X86ISD::PTEST, RHS, llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::SUB, llvm::X86ISD::TESTP, llvm::ISD::TRUNCATE, and llvm::ISD::XOR.

Referenced by combineSetCCEFLAGS().

combineSext()

static SDValue combineSext ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 55006 of file X86ISelLowering.cpp.

References combineExtSetcc(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineToExtendBoolVectorInReg(), combineToExtendCMOV(), DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isVector(), N, promoteExtBeforeAdd(), PromoteMaskArithmetic(), llvm::X86ISD::SETCC_CARRY, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, and llvm::ISD::TRUNCATE.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSextInRegCmov()

static SDValue combineSextInRegCmov ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 54732 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::X86ISD::CMOV, DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), N, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineSignExtendInReg().

combineShiftLeft()

static SDValue combineShiftLeft ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 49133 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, Cond, llvm::ISD::Constant, DL, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::EVT::isInteger(), llvm::EVT::isVector(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), N, llvm::X86ISD::SETCC_CARRY, llvm::ISD::SETUGE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, supportedVectorVarShift(), llvm::ISD::VSELECT, llvm::X86ISD::VSHLV, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineShiftRightArithmetic()

static SDValue combineShiftRightArithmetic ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 49201 of file X86ISelLowering.cpp.

References combineShiftToPMULH(), llvm::ISD::Constant, DL, llvm::APInt::eq(), llvm::SDNode::getAsAPIntVal(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDValue::hasOneUse(), llvm::APInt::isNegative(), llvm::EVT::isVector(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_UMin(), llvm::PatternMatch::m_Value(), N, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, Size, llvm::ISD::SRA, supportedVectorVarShift(), llvm::APInt::ult(), and llvm::X86ISD::VSRAV.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineShiftRightLogical()

static SDValue combineShiftRightLogical ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 49269 of file X86ISelLowering.cpp.

References llvm::ISD::AND, combineShiftToPMULH(), Cond, llvm::APInt::countr_one(), DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignificantBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::APInt::isMask(), llvm::isPowerOf2_32(), llvm::APInt::lshr(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), N, llvm::ISD::SETUGE, llvm::ISD::SETULT, llvm::ISD::SRL, supportedVectorVarShift(), llvm::ISD::VSELECT, and llvm::X86ISD::VSRLV.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineShiftToPMULH()

static SDValue combineShiftToPMULH ( SDNode *  N, SelectionDAG &  DAG, const SDLoc &  DL, const X86Subtarget &  Subtarget  )

static

Definition at line 49084 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), LHS, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, N, RHS, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.

Referenced by combineShiftRightArithmetic(), and combineShiftRightLogical().

combineShuffle()

static SDValue combineShuffle ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 42695 of file X86ISelLowering.cpp.

References canonicalizeShuffleWithOp(), combineShuffleOfConcatUndef(), combineShuffleToAddSubOrFMAddSub(), combineTargetShuffle(), combineToConsecutiveLoads(), combineX86ShufflesRecursively(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), isSoftF16(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), N, narrowShuffle(), and llvm::TargetLowering::SimplifyDemandedVectorElts().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineShuffleOfConcatUndef()

static SDValue combineShuffleOfConcatUndef ( SDNode *  N, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 42620 of file X86ISelLowering.cpp.

References Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::SDValue::isUndef(), and N.

Referenced by combineShuffle().

combineShuffleToAddSubOrFMAddSub()

static SDValue combineShuffleToAddSubOrFMAddSub ( SDNode *  N, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to combine a shuffle into a target-specific add-sub or mul-add-sub node.

Definition at line 42579 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADDSUB, combineShuffleToFMAddSub(), DL, llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMSUBADD, llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::is512BitVector(), isAddSubOrSubAdd(), isFMAddSubOrFMSubAdd(), and N.

Referenced by combineShuffle().

combineShuffleToFMAddSub()

static SDValue combineShuffleToFMAddSub ( SDNode *  N, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.

Definition at line 42537 of file X86ISelLowering.cpp.

References DL, llvm::ISD::FMA, llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMSUB, llvm::X86ISD::FMSUBADD, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDValue::hasOneUse(), isAddSubOrSubAddMask(), llvm::TargetLoweringBase::isTypeLegal(), N, std::swap(), and llvm::ISD::VECTOR_SHUFFLE.

Referenced by combineShuffleToAddSubOrFMAddSub().

combineSignExtendInReg()

static SDValue combineSignExtendInReg ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 54793 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), combineSextInRegCmov(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::X86Subtarget::hasInt256(), llvm::ISD::isNormalLoad(), llvm::ISD::LOAD, N, PromoteMaskArithmetic(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::SIGN_EXTEND_INREG.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSIntToFP()

static SDValue combineSIntToFP ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 56066 of file X86ISelLowering.cpp.

References assert(), llvm::BitWidth, llvm::X86TargetLowering::BuildFILD(), llvm::EVT::changeVectorElementType(), combineToFPTruncExtElt(), combineVectorCompareAndMaskUnaryOp(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86ISD::CVTSI2P, llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::X86Subtarget::getTargetLowering(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isNormalLoad(), llvm::MemSDNode::isSimple(), llvm::EVT::isVector(), llvm::ISD::LOAD, N, P, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SINT_TO_FP, llvm::X86ISD::STRICT_CVTSI2P, llvm::ISD::STRICT_SINT_TO_FP, and llvm::ISD::TRUNCATE.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineStore()

static SDValue combineStore ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 52612 of file X86ISelLowering.cpp.

References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::ISD::AND, llvm::ISD::ANY_EXTEND, CC, llvm::MVT::changeTypeToInteger(), llvm::X86ISD::CMOV, combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CSTORE, detectSSatPattern(), detectUSatPattern(), EmitTruncSStore(), llvm::ISD::EXTRACT_VECTOR_ELT, F, llvm::ISD::FABS, llvm::CallingConv::Fast, llvm::ISD::FNEG, llvm::MemSDNode::getAAInfo(), llvm::MemSDNode::getAddressSpace(), llvm::SelectionDAG::getAddrSpaceCast(), llvm::MemSDNode::getAlign(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::TypeSize::getFixed(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), GetOppositeBranchCondition(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::EVT::getStoreSize(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getTruncStore(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::MachinePointerInfo::getWithOffset(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::Hi, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MemSDNode::isNonTemporal(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::MemSDNode::isSimple(), llvm::EVT::isSimple(), llvm::StoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::Lo, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::SDNode::ops(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PEXTRW, llvm::X86AS::PTR32_SPTR, llvm::X86AS::PTR32_UPTR, llvm::X86AS::PTR64, llvm::ISD::SCALAR_TO_VECTOR, scalarizeVectorStore(), splitVectorStore(), llvm::ISD::TokenFactor, llvm::ISD::TRUNCATE, llvm::Align::value(), llvm::X86ISD::VTRUNC, llvm::X86ISD::VTRUNCS, llvm::X86ISD::VTRUNCUS, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSub()

static SDValue combineSub ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 57050 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADC, llvm::ISD::ADD, assert(), combineAddOrSubToADCOrSBB(), combineSubABS(), combineSubSetcc(), combineToHorizontalAddSub(), combineXorSubCTLZ(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getVTList(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDNode::hasOneUse(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::isNullConstant(), llvm::X86::isZeroNode(), N, llvm::X86ISD::SBB, llvm::ISD::SUB, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineSubABS()

static SDValue combineSubABS ( EVT  VT, const SDLoc &  DL, SDValue  N0, SDValue  N1, SelectionDAG &  DAG  )

static

Definition at line 56948 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), CC, llvm::X86ISD::CMOV, Cond, llvm::X86::COND_B, llvm::X86::COND_L, llvm::X86::COND_NS, llvm::X86::COND_S, DL, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), llvm::ISD::SUB, llvm::X86ISD::SUB, and X.

Referenced by combineSub().

combineSubSetcc()

static SDValue combineSubSetcc ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 56997 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADD, CC, DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), N, llvm::X86ISD::SETCC, and llvm::ISD::ZERO_EXTEND.

Referenced by combineSub().

combineTargetShuffle()

static SDValue combineTargetShuffle ( SDValue  N, const SDLoc &  DL, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Try to combine x86 target specific shuffles.

Definition at line 41613 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::X86ISD::BLENDI, llvm::CallingConv::C, canonicalizeLaneShuffleWithRepeatedOps(), collectConcatOps(), combineCommutableSHUFP(), combineRedundantDWordShuffle(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::ISD::CONCAT_VECTORS, D, DL, extract128BitVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::ISD::FADD, llvm::ISD::FDIV, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::ConstantVector::get(), llvm::MemSDNode::getBasePtr(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), getConstVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::TypeSize::getFixed(), llvm::MVT::getFixedSizeInBits(), llvm::MachineMemOperand::getFlags(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::User::getOperand(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), getPSHUFShuffleMask(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleAndZeroables(), getTargetShuffleInputs(), getTargetShuffleMask(), llvm::EVT::getTypeForEVT(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::MachinePointerInfo::getWithOffset(), getZeroVector(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::Value::hasOneUse(), Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::X86ISD::INSERTPS, llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::MVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isEXTLoad(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), isShuffleFoldableLoad(), llvm::MemSDNode::isSimple(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), isUndefOrInRange(), llvm::ISD::isUNINDEXEDLoad(), llvm::VectorType::isValidElementType(), llvm::EVT::isVector(), LHS, llvm::SelectionDAG::MaskedValueIsZero(), llvm::X86::MaxShuffleCombineDepth, llvm::MachineMemOperand::MOLoad, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSS, N, narrowLoadToVZLoad(), llvm::narrowShuffleMaskElts(), llvm::Offset, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, Ptr, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::SCALAR_TO_VECTOR, llvm::APIntOps::ScaleBitMask(), scaleShuffleElements(), llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::SmallVectorBase< Size_T >::size(), Size, llvm::SM_SentinelUndef, llvm::ISD::SRL, std::swap(), llvm::ISD::TRUNCATE, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, llvm::X86ISD::VSRLI, llvm::X86ISD::VZEXT_LOAD, llvm::X86ISD::VZEXT_MOVL, and widenSubVector().

Referenced by combineShuffle().

combineTESTP()

static SDValue combineTESTP ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 55711 of file X86ISelLowering.cpp.

References llvm::APInt::getAllOnes(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), N, and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineToConsecutiveLoads()

static SDValue combineToConsecutiveLoads ( EVT  VT, SDValue  Op, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, bool  IsAfterLegalize  )

static

Definition at line 7341 of file X86ISelLowering.cpp.

References assert(), DL, EltsFromConsecutiveLoads(), getShuffleScalarElt(), llvm::EVT::getVectorNumElements(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by combineShuffle().

combineToExtendBoolVectorInReg()

static SDValue combineToExtendBoolVectorInReg ( unsigned  Opcode, const SDLoc &  DL, EVT  VT, SDValue  N0, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 46377 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::SmallVectorImpl< T >::append(), assert(), llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitsSet(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSplat(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::EVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isScalarInteger(), llvm::EVT::isVector(), llvm::Offset, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SETEQ, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.

Referenced by combineSelect(), combineSext(), and combineZext().

combineToExtendCMOV()

static SDValue combineToExtendCMOV ( SDNode *  Extend, SelectionDAG &  DAG  )

static

Definition at line 54915 of file X86ISelLowering.cpp.

References llvm::X86ISD::CMOV, DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), and llvm::ISD::SIGN_EXTEND.

Referenced by combineSext(), and combineZext().

combineToFPTruncExtElt()

static SDValue combineToFPTruncExtElt ( SDNode *  N, SelectionDAG &  DAG  )

static

If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector element with a bitcast.

This tries to keep the sequence on XMM registers rather than moving between vector and GPRs.

Definition at line 55970 of file X86ISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), N, and llvm::ISD::TRUNCATE.

Referenced by combineSIntToFP().

combineToHorizontalAddSub()

static SDValue combineToHorizontalAddSub ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 53169 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SmallVectorBase< Size_T >::empty(), llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, isHorizontalBinOp(), LHS, N, RHS, SplitOpsAndApply(), llvm::ISD::SUB, and llvm::ISD::VECTOR_SHUFFLE.

Referenced by combineAdd(), combineFaddFsub(), and combineSub().

combineTruncate()

static SDValue combineTruncate ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 53691 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, combinePMULH(), combineTruncatedArithmetic(), combineTruncateWithSat(), detectPMADDUBSW(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::ISD::LRINT, llvm::X86ISD::MMX_MOVD2W, and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineTruncatedArithmetic()

static SDValue combineTruncatedArithmetic ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, const SDLoc &  DL  )

static

Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen.

e.g. TRUNC( BINOP( X, Y ) ) --> BINOP( TRUNC( X ), TRUNC( Y ) ) TODO: This overlaps with the generic combiner's visitTRUNCATE. Remove anything that is guaranteed to be transformed by DAGCombiner.

Definition at line 53389 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::isBuildVectorOfConstantSDNodes(), IsFreeTruncation, llvm::TargetLoweringBase::isOperationLegal(), llvm::EVT::isVector(), llvm::ISD::MUL, N, llvm::ISD::OR, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by combineTruncate().

combineTruncateWithSat()

static SDValue combineTruncateWithSat ( SDValue  In, EVT  VT, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 52048 of file X86ISelLowering.cpp.

References assert(), llvm::EVT::changeVectorElementType(), llvm::ISD::CONCAT_VECTORS, detectSSatPattern(), detectUSatPattern(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is512BitVector(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, truncateVectorWithPACK(), llvm::X86Subtarget::useAVX512Regs(), llvm::X86ISD::VTRUNCS, and llvm::X86ISD::VTRUNCUS.

Referenced by combineTruncate().

combineUIntToFP()

static SDValue combineUIntToFP ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 56002 of file X86ISelLowering.cpp.

References llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), N, P, llvm::SelectionDAG::SignBitIsZero(), llvm::ISD::SINT_TO_FP, llvm::ISD::STRICT_SINT_TO_FP, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorCompare()

static SDValue combineVectorCompare ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 57112 of file X86ISelLowering.cpp.

References assert(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getScalarSizeInBits(), getTargetConstantBitsFromNode(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZero(), I, LHS, N, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, Results, and RHS.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorCompareAndMaskUnaryOp()

static SDValue combineVectorCompareAndMaskUnaryOp ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 55912 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::BuildVectorSDNode::isConstant(), llvm::EVT::isVector(), and N.

Referenced by combineSIntToFP().

combineVectorHADDSUB()

static SDValue combineVectorHADDSUB ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 49640 of file X86ISelLowering.cpp.

References assert(), combineHorizOpWithShuffle(), DL, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorVT(), llvm::X86ISD::HADD, llvm::X86ISD::HSUB, llvm::SDValue::isUndef(), LHS, N, llvm::X86ISD::PSHUFD, RHS, and shouldUseHorizontalOp().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorInsert()

static SDValue combineVectorInsert ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 49874 of file X86ISelLowering.cpp.

References assert(), combineX86ShufflesRecursively(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), Idx, llvm::ISD::INSERT_VECTOR_ELT, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::isNullConstant(), llvm::EVT::isSimple(), llvm::SDValue::isUndef(), N, llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::ISD::SCALAR_TO_VECTOR, and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorPack()

static SDValue combineVectorPack ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 49489 of file X86ISelLowering.cpp.

References assert(), combineHorizOpWithShuffle(), combineX86ShufflesRecursively(), llvm::SelectionDAG::ComputeNumSignBits(), Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), getConstVector(), getEXTEND_VECTOR_INREG(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX512(), llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::APInt::isIntN(), llvm::APInt::isNegative(), IsNOT(), llvm::SDValue::isUndef(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::APInt::setBit(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::APInt::trunc(), llvm::ISD::TRUNCATE, llvm::APInt::truncSSat(), llvm::X86ISD::VTRUNC, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorShiftImm()

static SDValue combineVectorShiftImm ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 49721 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::APInt::ashrInPlace(), assert(), llvm::CallingConv::C, combineX86ShufflesRecursively(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), getConstVector(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getV4X86ShuffleImm(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::ISD::isBitwiseLogicOp(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDNode::isOnlyUserOf(), llvm::SDValue::isUndef(), LHS, llvm::APInt::lshrInPlace(), N, llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFD, RHS, llvm::TargetLowering::SimplifyDemandedBits(), llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and X.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorShiftVar()

static SDValue combineVectorShiftVar ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 49688 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::EVT::getSimpleVT(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorAllZeros(), N, llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::X86ISD::VSHL, llvm::X86ISD::VSRA, and llvm::X86ISD::VSRL.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVectorSizedSetCCEquality()

static SDValue combineVectorSizedSetCCEquality ( EVT  VT, SDValue  X, SDValue  Y, ISD::CondCode  CC, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Try to map a 128-bit or larger integer comparison to vector instructions before type legalization splits it up into chunks.

Definition at line 22385 of file X86ISelLowering.cpp.

References assert(), CC, llvm::X86::COND_E, llvm::X86::COND_NE, DL, emitOrXorXorTree(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getScalarValueSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::X86Subtarget::hasAVX(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::INSERT_SUBVECTOR, llvm::isNullConstant(), isOrXorXorTree(), llvm::EVT::isScalarInteger(), llvm::ISD::LOAD, llvm::X86ISD::MOVMSK, llvm::peekThroughBitcasts(), llvm::X86ISD::PTEST, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::TRUNCATE, llvm::X86Subtarget::useAVX512Regs(), X, llvm::ISD::XOR, Y, and llvm::ISD::ZERO_EXTEND.

Referenced by combineSetCC().

combineVEXTRACT_STORE()

static SDValue combineVEXTRACT_STORE ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 52945 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::DELETED_NODE, llvm::APInt::getLowBitsSet(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), N, and llvm::TargetLowering::SimplifyDemandedVectorElts().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVPDPBUSDPattern()

static SDValue combineVPDPBUSDPattern ( SDNode *  Extract, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 45510 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, createVPDPBUSD(), detectExtMul(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::isPowerOf2_32(), LHS, llvm::Log2_32(), llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::MUL, and RHS.

Referenced by combineExtractVectorElt().

combineVPMADD()

static SDValue combineVPMADD ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 58600 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getScalarSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::Hi, I, llvm::ISD::isBuildVectorAllZeros(), LHS, llvm::Lo, N, RHS, llvm::APInt::sext(), llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::VPMADDUBSW, llvm::X86ISD::VPMADDWD, and llvm::APInt::zext().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineVSelectToBLENDV()

static SDValue combineVSelectToBLENDV ( SDNode *  N, SelectionDAG &  DAG, const SDLoc &  DL, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element.

This function will also call SimplifyDemandedBits on already created BLENDV to perform additional simplifications.

Definition at line 46686 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::BitWidth, llvm::X86ISD::BLENDV, llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), Cond, DL, llvm::SelectionDAG::getNode(), llvm::Use::getOperandNo(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Use::getUser(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is128BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::TargetLowering::SimplifyDemandedBits(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), and llvm::ISD::VSELECT.

Referenced by combineSelect().

combineVSelectWithAllOnesOrZeros()

static SDValue combineVSelectWithAllOnesOrZeros ( SDNode *  N, SelectionDAG &  DAG, const SDLoc &  DL, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation.

TODO: Move to DAGCombiner, possibly using TargetLowering::hasAndNot()?

Definition at line 46476 of file X86ISelLowering.cpp.

References llvm::And, llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), CC, llvm::SelectionDAG::ComputeNumSignBits(), Cond, DL, llvm::get(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LHS, N, llvm::Or, llvm::ISD::OR, RHS, llvm::ISD::SETCC, std::swap(), and llvm::ISD::VSELECT.

Referenced by combineSelect().

combineVTRUNC()

static SDValue combineVTRUNC ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 53729 of file X86ISelLowering.cpp.

References detectSSatPattern(), detectUSatPattern(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), N, llvm::TargetLowering::SimplifyDemandedBits(), llvm::X86ISD::VTRUNCS, and llvm::X86ISD::VTRUNCUS.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combinevXi1ConstantToInteger()

static SDValue combinevXi1ConstantToInteger ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 44637 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), Idx, and llvm::ISD::isBuildVectorOfConstantSDNodes().

Referenced by combineBitcast(), combineCastedMaskArithmetic(), combineSelect(), and combineStore().

combineX86AddSub()

static SDValue combineX86AddSub ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  ST  )

static

Definition at line 56396 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::X86ISD::ADD, assert(), combineAddOrSubToADCOrSBB(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineX86SubCmpForFlags(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getVTList(), llvm::isOneConstant(), LHS, N, RHS, llvm::ISD::SUB, and llvm::X86ISD::SUB.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineX86CloadCstore()

static SDValue combineX86CloadCstore ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 57024 of file X86ISelLowering.cpp.

References llvm::X86::COND_NE, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::isZeroNode(), N, llvm::X86ISD::SETCC, and llvm::X86ISD::SUB.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineX86GatherScatter()

static SDValue combineX86GatherScatter ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 55726 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::DELETED_NODE, llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), N, and llvm::TargetLowering::SimplifyDemandedBits().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineX86INT_TO_FP()

static SDValue combineX86INT_TO_FP ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 54459 of file X86ISelLowering.cpp.

References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::ISD::isNormalLoad(), N, narrowLoadToVZLoad(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and llvm::TargetLowering::SimplifyDemandedVectorElts().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineX86SetCC()

static SDValue combineX86SetCC ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 55879 of file X86ISelLowering.cpp.

References CC, combineSetCCEFLAGS(), DL, getSETCC(), and N.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineX86ShuffleChain()

static SDValue combineX86ShuffleChain ( ArrayRef< SDValue >  Inputs, SDValue  Root, ArrayRef< int >  BaseMask, int  Depth, bool  HasVariableMask, bool  AllowVariableCrossLaneMask, bool  AllowVariablePerLaneMask, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Combine an arbitrary chain of shuffles into a single instruction if possible.

This is the leaf of the recursive combine below. When we have found some chain of single-use x86 shuffle instructions and accumulated the combined shuffle mask represented by them, this will try to pattern match that mask into either a single instruction if there is a special purpose instruction for this operation, or into a PSHUFB instruction which is a fully general instruction but should only be used to replace chains over a certain depth.

Definition at line 39370 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::AllOnes, llvm::ISD::AND, assert(), combineX86ShuffleChainWithExtract(), llvm::ISD::CONCAT_VECTORS, llvm::Depth, DL, llvm::ArrayRef< T >::empty(), extract128BitVector(), extractSubVector(), llvm::X86ISD::EXTRQI, llvm::X86ISD::FAND, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::X86ISD::INSERTPS, llvm::X86ISD::INSERTQI, insertSubVector(), is128BitLaneCrossingShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isAnyZero(), llvm::MVT::isFloatingPoint(), isInRange(), isSequentialOrUndefInRange(), isSequentialOrUndefOrZeroInRange(), llvm::SelectionDAG::isSplatValue(), isTargetShuffleEquivalent(), llvm::TargetLoweringBase::isTypeLegal(), isUndef(), isUndefOrEqual(), isUndefOrInRange(), isUndefOrZero(), isUndefOrZeroInRange(), llvm::MVT::isVector(), LHS, llvm::Lo, lowerShuffleWithPERMV(), matchBinaryPermuteShuffle(), matchBinaryShuffle(), matchShuffleAsEXTRQ(), matchShuffleAsInsertPS(), matchShuffleAsINSERTQ(), matchShuffleAsVTRUNC(), matchUnaryPermuteShuffle(), matchUnaryShuffle(), llvm::X86::mayFoldLoad(), llvm::narrowShuffleMaskElts(), OpIndex, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveZeroablesFromTargetShuffle(), RHS, llvm::ISD::SCALAR_TO_VECTOR, scaleShuffleElements(), llvm::APInt::setBit(), llvm::SelectionDAG::shouldOptForSize(), llvm::X86ISD::SHUF128, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::TRUNCATE, llvm::X86Subtarget::useAVX512Regs(), llvm::SDNode::user_begin(), llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, llvm::ISD::VSELECT, llvm::X86ISD::VTRUNC, and widenSubVector().

Referenced by combineX86ShuffleChainWithExtract(), and combineX86ShufflesRecursively().

combineX86ShuffleChainWithExtract()

static SDValue combineX86ShuffleChainWithExtract ( ArrayRef< SDValue >  Inputs, SDValue  Root, ArrayRef< int >  BaseMask, int  Depth, bool  HasVariableMask, bool  AllowVariableCrossLaneMask, bool  AllowVariablePerLaneMask, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 40121 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), canonicalizeShuffleMaskWithCommute(), canWidenShuffleElements(), combineX86ShuffleChain(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::Depth, llvm::SmallVectorBase< Size_T >::empty(), llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), I, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::isNullConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::peekThroughBitcasts(), resolveTargetShuffleInputsAndMask(), llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, and std::swap().

Referenced by combineX86ShuffleChain(), and combineX86ShufflesRecursively().

combineX86ShufflesConstants()

static SDValue combineX86ShufflesConstants ( MVT  VT, ArrayRef< SDValue >  Ops, ArrayRef< int >  Mask, bool  HasVariableMask, SelectionDAG &  DAG, const SDLoc &  DL, const X86Subtarget &  Subtarget  )

static

Definition at line 40473 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), getZeroVector(), I, llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::none_of(), llvm::APInt::setBit(), llvm::SelectionDAG::shouldOptForSize(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineTargetShuffle(), and combineX86ShufflesRecursively().

combineX86ShufflesRecursively() [1/2]

static SDValue combineX86ShufflesRecursively ( ArrayRef< SDValue >  SrcOps, int  SrcOpIndex, SDValue  Root, ArrayRef< int >  RootMask, ArrayRef< const SDNode * >  SrcNodes, unsigned  Depth, unsigned  MaxDepth, bool  HasVariableMask, bool  AllowVariableCrossLaneMask, bool  AllowVariablePerLaneMask, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Fully generic combining of x86 shuffle instructions.

This should be the last combine run over the x86 shuffle instructions. Once they have been fully optimized, this will recursively consider all chains of single-use shuffle instructions, build a generic model of the cumulative shuffle operation, and check for simpler instructions which implement this operation. We use this primarily for two purposes:

1) Collapse generic shuffles to specialized single instructions when equivalent. In most cases, this is just an encoding size win, but sometimes we will collapse multiple generic shuffles into a single special-purpose shuffle. 2) Look for sequences of shuffle instructions with 3 or more total instructions, and replace them with the slightly more expensive SSSE3 PSHUFB instruction if available. We do this as the last combining step to ensure we avoid using PSHUFB if we can implement the shuffle with a suitable short sequence of other instructions. The PSHUFB will either use a register or have to read from memory and so is slightly (but only slightly) more expensive than the other shuffle instructions.

Because this is inherently a quadratic operation (for each shuffle in a chain, we recurse up the chain), the depth is limited to 8 instructions. This should never be an issue in practice as the shuffle lowering doesn't produce sequences of more than 8 instructions.

FIXME: We will currently miss some cases where the redundant shuffling would simplify under the threshold for PSHUFB formation because of combine-ordering. To fix this, we should do the redundant instruction combining in this recursive walk.

Definition at line 40595 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::any_of(), llvm::SmallVectorImpl< T >::append(), llvm::SDNode::areOnlyUsersOf(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), canonicalizeShuffleMaskWithCommute(), canonicalizeShuffleMaskWithHorizOp(), canWidenShuffleElements(), combineX86ShuffleChain(), combineX86ShuffleChainWithExtract(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::countr_zero(), llvm::Depth, DL, llvm::ArrayRef< T >::empty(), llvm::SmallVectorBase< Size_T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::enumerate(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::extractBits(), llvm::APInt::extractBits(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::SelectionDAG::getBitcast(), getNode(), getOnesVector(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::Hi, I, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::is_contained(), llvm::APInt::isAllOnes(), llvm::ISD::isBuildVectorAllOnes(), isInRange(), llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::EVT::isSimple(), isTargetShuffleVariableMask(), isUndefOrZero(), llvm::EVT::isVector(), llvm::MVT::isVector(), llvm::APInt::isZero(), LHS, llvm::Lo, MaxDepth, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveTargetShuffleFromZeroables(), resolveTargetShuffleInputsAndMask(), RHS, llvm::APIntOps::ScaleBitMask(), llvm::APInt::setBit(), llvm::TargetLowering::SimplifyMultipleUseDemandedVectorElts(), llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, std::swap(), llvm::APInt::trunc(), widenSubVector(), and llvm::APInt::zext().

Referenced by combineAnd(), combineAndnp(), combineEXTEND_VECTOR_INREG(), combineINSERT_SUBVECTOR(), combineOr(), combineShuffle(), combineTargetShuffle(), combineVectorInsert(), combineVectorPack(), combineVectorShiftImm(), combineX86ShufflesRecursively(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

combineX86ShufflesRecursively() [2/2]

static SDValue combineX86ShufflesRecursively ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Helper entry wrapper to combineX86ShufflesRecursively.

Definition at line 41032 of file X86ISelLowering.cpp.

References combineX86ShufflesRecursively(), and llvm::X86::MaxShuffleCombineDepth.

combineX86SubCmpForFlags()

static SDValue combineX86SubCmpForFlags ( SDNode *  N, SDValue  Flag, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  ST  )

static

Definition at line 50743 of file X86ISelLowering.cpp.

References llvm::X86ISD::BRCOND, CC, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::X86::COND_NE, llvm::SDNode::getAsAPIntVal(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::APInt::getSExtValue(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), llvm::isNullConstant(), llvm::isOneConstant(), llvm_unreachable, N, llvm::SDNode::op_values(), llvm::X86ISD::SETCC, llvm::X86ISD::SUB, and X.

Referenced by combineCMP(), and combineX86AddSub().

combineXor()

static SDValue combineXor ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 54108 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineBMILogicOp(), combineFneg(), combineOrXorWithSETCC(), combineXorSubCTLZ(), convertIntLogicToFPLogic(), DL, foldVectorXorShiftIntoCmp(), foldXor1SetCC(), foldXorTruncShiftIntoCmp(), llvm::X86ISD::FXOR, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllOnes(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), LHS, N, RHS, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

combineXorSubCTLZ()

static SDValue combineXorSubCTLZ ( SDNode *  N, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 54052 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::BSR, llvm::CallingConv::C, llvm::ISD::CTLZ_ZERO_UNDEF, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), N, llvm::ISD::SUB, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by combineSub(), and combineXor().

combineZext()

static SDValue combineZext ( SDNode *  N, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

Definition at line 55226 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, combineExtSetcc(), combineOrCmpEqZeroToCtlzSrl(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineToExtendBoolVectorInReg(), combineToExtendCMOV(), concatSubVectors(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::X86ISD::PACKUS, promoteExtBeforeAdd(), PromoteMaskArithmetic(), llvm::X86ISD::SETCC_CARRY, and llvm::ISD::TRUNCATE.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

commuteSelect()

static SDValue commuteSelect ( SDNode *  N, SelectionDAG &  DAG, const SDLoc &  DL, const X86Subtarget &  Subtarget  )

static

Definition at line 46839 of file X86ISelLowering.cpp.

References canCombineAsMaskOperation(), Cond, DL, llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::X86Subtarget::hasAVX512(), LHS, N, RHS, llvm::ISD::SETCC, and llvm::ISD::VSELECT.

Referenced by combineSelect().

computeInLaneShuffleMask()

static void computeInLaneShuffleMask ( const ArrayRef< int > &  Mask, int  LaneSize, SmallVector< int > &  InLaneMask  )

static

Helper to get compute inlane shuffle mask for a complete shuffle mask.

Definition at line 15095 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::assign(), and Size.

Referenced by lowerShuffleAsLanePermuteAndShuffle(), and lowerV8F32Shuffle().

computeKnownBitsForHorizontalOperation()

static KnownBits computeKnownBitsForHorizontalOperation ( const SDValue  Op, const APInt &  DemandedElts, unsigned  Depth, const SelectionDAG &  DAG, const function_ref< KnownBits(const KnownBits &, const KnownBits &)>  KnownBitsFunc  )

static

Definition at line 38034 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::Depth, llvm::getHorizDemandedEltsForFirstOperand(), and llvm::APInt::isZero().

computeKnownBitsForPMADDUBSW()

static void computeKnownBitsForPMADDUBSW ( SDValue  LHS, SDValue  RHS, KnownBits &  Known, const APInt &  DemandedElts, const SelectionDAG &  DAG, unsigned  Depth  )

static

Definition at line 38011 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::Depth, llvm::APInt::getSplat(), llvm::Hi, LHS, llvm::Lo, llvm::KnownBits::mul(), RHS, llvm::KnownBits::sadd_sat(), llvm::APIntOps::ScaleBitMask(), llvm::KnownBits::sext(), and llvm::KnownBits::zext().

Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode().

computeKnownBitsForPMADDWD()

static void computeKnownBitsForPMADDWD ( SDValue  LHS, SDValue  RHS, KnownBits &  Known, const APInt &  DemandedElts, const SelectionDAG &  DAG, unsigned  Depth  )

static

Definition at line 37989 of file X86ISelLowering.cpp.

References llvm::KnownBits::add(), llvm::SelectionDAG::computeKnownBits(), llvm::Depth, llvm::APInt::getSplat(), llvm::Hi, LHS, llvm::Lo, llvm::KnownBits::mul(), RHS, llvm::APIntOps::ScaleBitMask(), and llvm::KnownBits::sext().

Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode().

computeKnownBitsForPSADBW()

static void computeKnownBitsForPSADBW ( SDValue  LHS, SDValue  RHS, KnownBits &  Known, const APInt &  DemandedElts, const SelectionDAG &  DAG, unsigned  Depth  )

static

Definition at line 37972 of file X86ISelLowering.cpp.

References llvm::KnownBits::abdu(), llvm::KnownBits::add(), llvm::SelectionDAG::computeKnownBits(), llvm::Depth, LHS, RHS, llvm::APIntOps::ScaleBitMask(), and llvm::KnownBits::zext().

Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode().

computeZeroableShuffleElements()

static void computeZeroableShuffleElements ( ArrayRef< int >  Mask, SDValue  V1, SDValue  V2, APInt &  KnownUndef, APInt &  KnownZero  )

static

Compute whether each element of a shuffle is zeroable.

A "zeroable" vector shuffle element is one which can be lowered to zero. Either it is an undef element in the shuffle mask, the element of the input referenced is undef, or the element of the input referenced is known to be zero. Many x86 shuffles can zero lanes cheaply and we often want to handle as many lanes with this technique as possible to simplify the remaining shuffle.

Definition at line 5687 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::APInt::extractBits(), llvm::SDValue::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::APInt::getZero(), llvm::ISD::isBuildVectorAllZeros(), llvm::X86::isZeroNode(), llvm::peekThroughBitcasts(), llvm::APInt::setBit(), and Size.

Referenced by lowerVECTOR_SHUFFLE().

concatSubVectors()

static SDValue concatSubVectors ( SDValue  V1, SDValue  V2, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Definition at line 4594 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getContext(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), and insertSubVector().

Referenced by combineAdd(), combineConcatVectorOps(), combineZext(), EltsFromConsecutiveLoads(), and lowerV8I16Shuffle().

convertIntLogicToFPLogic()

static SDValue convertIntLogicToFPLogic ( unsigned  Opc, const SDLoc &  DL, EVT  VT, SDValue  N0, SDValue  N1, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

If both input operands of a logic op are being cast from floating-point types or FP compares, try to convert this into a floating-point logic node to avoid unnecessary moves from SSE to integer registers.

Definition at line 50235 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, cheapX86FSETCC_SSE(), convertIntLogicToFPLogicOpcode(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::OR, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SETCC, and llvm::ISD::XOR.

Referenced by combineAnd(), combineOr(), and combineXor().

convertIntLogicToFPLogicOpcode()

static unsigned convertIntLogicToFPLogicOpcode ( unsigned  Opcode )

static

Definition at line 50219 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm_unreachable, llvm::ISD::OR, and llvm::ISD::XOR.

Referenced by combineBitOpWithMOVMSK(), and convertIntLogicToFPLogic().

convertShiftLeftToScale()

static SDValue convertShiftLeftToScale ( SDValue  Amt, const SDLoc &  dl, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 29952 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, convertShiftLeftToScale(), llvm::ISD::FP_TO_SINT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPack(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, I, llvm::Lo, llvm::X86ISD::PACKUS, llvm::APInt::shl(), and llvm::ISD::SHL.

Referenced by convertShiftLeftToScale(), LowerRotate(), and LowerShift().

createMMXBuildVector()

static SDValue createMMXBuildVector ( BuildVectorSDNode *  BV, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 44710 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::append(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::BuildVectorSDNode::getSplatValue(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasSSE1(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::X86ISD::MMX_MOVW2D, llvm::X86ISD::MOVDQ2Q, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SmallVectorBase< Size_T >::size(), and llvm::Splat.

Referenced by combineBitcast().

createPackShuffleMask()

static void createPackShuffleMask ( MVT  VT, SmallVectorImpl< int > &  Mask, bool  Unary, unsigned  NumStages = 1  )

static

Create a shuffle mask that matches the PACKSS/PACKUS truncation.

A multi-stage pack shuffle mask is created by specifying NumStages > 1. Note: This ignores saturation, so inputs must be checked first.

Definition at line 5303 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), and llvm::Offset.

Referenced by getFauxShuffleMask(), and matchShuffleWithPACK().

createPHIsForCMOVsInSinkBB()

static MachineInstrBuilder createPHIsForCMOVsInSinkBB ( MachineBasicBlock::iterator  MIItBegin, MachineBasicBlock::iterator  MIItEnd, MachineBasicBlock *  TrueMBB, MachineBasicBlock *  FalseMBB, MachineBasicBlock *  SinkMBB  )

static

Definition at line 35617 of file X86ISelLowering.cpp.

References llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), CC, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::TargetSubtargetInfo::getInstrInfo(), llvm::X86::GetOppositeBranchCondition(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getSubtarget(), std::swap(), and TII.

createPSADBW()

static SDValue createPSADBW ( SelectionDAG &  DAG, const SDValue &  Zext0, const SDValue &  Zext1, const SDLoc &  DL, const X86Subtarget &  Subtarget  )

static

Definition at line 45267 of file X86ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), llvm::X86ISD::PSADBW, RegSize, and SplitOpsAndApply().

Referenced by combineBasicSADPattern().

createSetFPEnvNodes()

static SDValue createSetFPEnvNodes ( SDValue  Ptr, SDValue  Chain, const SDLoc &  DL, EVT  MemVT, MachineMemOperand *  MMO, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 28371 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::X86ISD::FLDENVm, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasSSE1(), llvm::ISD::INTRINSIC_VOID, Ptr, and X87StateSize.

createShuffleMaskFromVSELECT()

static bool createShuffleMaskFromVSELECT ( SmallVectorImpl< int > &  Mask, SDValue  Cond, bool  IsBLENDV = false  )

static

Definition at line 5905 of file X86ISelLowering.cpp.

References Cond, llvm::EVT::getScalarSizeInBits(), getTargetConstantBitsFromNode(), llvm::EVT::getVectorNumElements(), isZero(), and llvm::SM_SentinelUndef.

Referenced by combineSelect(), getFauxShuffleMask(), and lowerVSELECTtoVectorShuffle().

createVariablePermute()

static SDValue createVariablePermute ( MVT  VT, SDValue  SrcVec, SDValue  IndicesVec, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source values and a vector of extraction indices.

The vectors might be manipulated to match the type of the permute op.

Definition at line 8622 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::EVT::changeVectorElementTypeToInteger(), createVariablePermute(), DL, extractSubVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getSelectCC(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), Idx, llvm::isPowerOf2_64(), llvm::ISD::MUL, llvm::X86ISD::PSHUFB, llvm::ISD::SETEQ, llvm::MVT::SimpleTy, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, widenSubVector(), and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by createVariablePermute(), and LowerBUILD_VECTORAsVariablePermute().

createVPDPBUSD()

static SDValue createVPDPBUSD ( SelectionDAG &  DAG, SDValue  LHS, SDValue  RHS, unsigned &  LogBias, const SDLoc &  DL, const X86Subtarget &  Subtarget  )

static

Definition at line 45221 of file X86ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), LHS, RegSize, RHS, SplitOpsAndApply(), and llvm::X86ISD::VPDPBUSD.

Referenced by combineVPDPBUSDPattern().

detectExtMul()

static bool detectExtMul ( SelectionDAG &  DAG, const SDValue &  Mul, SDValue &  Op0, SDValue &  Op1  )

static

Definition at line 45170 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::KnownBits::countMaxActiveBits(), llvm::SDValue::getOpcode(), llvm::BuildVectorSDNode::isConstant(), IsFreeTruncation, llvm::Mul, llvm::ISD::SIGN_EXTEND, std::swap(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineVPDPBUSDPattern().

detectPMADDUBSW()

static SDValue detectPMADDUBSW ( SDValue  In, EVT  VT, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, const SDLoc &  DL  )

static

Definition at line 53549 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::ISD::BUILD_VECTOR, detectSSatPattern(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSSE3(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::ISD::MUL, llvm::ISD::SIGN_EXTEND, SplitOpsAndApply(), std::swap(), llvm::X86ISD::VPMADDUBSW, and llvm::ISD::ZERO_EXTEND.

Referenced by combineTruncate().

detectSSatPattern()

static SDValue detectSSatPattern ( SDValue  In, EVT  VT, bool  MatchPackUS = false  )

static

Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type).

With MatchPackUS, the smax/smin range is [0, unsigned_max_of_dest_type]. Return the source value to be truncated or SDValue() if the pattern was not matched.

Definition at line 52021 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::getAllOnes(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::APInt::getZero(), llvm::PatternMatch::m_SMax(), llvm::PatternMatch::m_SMin(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Value(), llvm::APInt::sext(), llvm::SMax, llvm::SMin, and llvm::APInt::zext().

Referenced by combineStore(), combineTruncateWithSat(), combineVTRUNC(), and detectPMADDUBSW().

detectUSatPattern()

static SDValue detectUSatPattern ( SDValue  In, EVT  VT, SelectionDAG &  DAG, const SDLoc &  DL  )

static

Detect patterns of truncation with unsigned saturation:

1. (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type). Return the source value x to be truncated or SDValue() if the pattern was not matched.

2. (truncate (smin (smax (x, C1), C2)) to dest_type), where C1 >= 0 and C2 is unsigned max of destination type.

   (truncate (smax (smin (x, C2), C1)) to dest_type) where C1 >= 0, C2 is unsigned max of destination type and C1 <= C2.

   These two patterns are equivalent to: (truncate (umin (smax(x, C1), unsigned_max_of_dest_type)) to dest_type) So return the smax(x, C1) value to be truncated or SDValue() if the pattern was not matched.

Definition at line 51981 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::isMask(), llvm::APInt::isNonNegative(), llvm::PatternMatch::m_SMax(), llvm::PatternMatch::m_SMin(), llvm::PatternMatch::m_UMin(), llvm::PatternMatch::m_Value(), llvm::SMax, llvm::ISD::SMAX, llvm::SMin, llvm::APInt::uge(), and llvm::UMin.

Referenced by combineStore(), combineTruncateWithSat(), and combineVTRUNC().

detectZextAbsDiff()

static bool detectZextAbsDiff ( const SDValue &  Abs, SDValue &  Op0, SDValue &  Op1  )

static

Definition at line 45203 of file X86ISelLowering.cpp.

References llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::SUB, and llvm::ISD::ZERO_EXTEND.

Referenced by combineBasicSADPattern().

EltsFromConsecutiveLoads()

static SDValue EltsFromConsecutiveLoads ( EVT  VT, ArrayRef< SDValue >  Elts, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, bool  IsAfterLegalize  )

static

Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.

Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a

Definition at line 7059 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::sampleprof::Base, concatSubVectors(), llvm::APInt::countr_zero(), DL, llvm::ArrayRef< T >::drop_back(), EltsFromConsecutiveLoads(), llvm::APInt::extractBits(), findEltLoadSrc(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::MemSDNode::getAlign(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineMemOperand::getFlags(), llvm::EVT::getFloatingPointVT(), llvm::MVT::getFloatingPointVT(), llvm::EVT::getIntegerVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::EVT::getStoreSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::APInt::isAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::MachinePointerInfo::isDereferenceable(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::MemSDNode::isNonTemporal(), llvm::TargetLoweringBase::isOperationLegal(), llvm::isPowerOf2_32(), llvm::MemSDNode::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), isUndef(), llvm::EVT::isVector(), llvm::APInt::isZero(), llvm::X86::isZeroNode(), llvm::ISD::LOAD, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::Match, llvm::X86::mayFoldLoadIntoBroadcastFromMem(), llvm::MachineMemOperand::MOLoad, llvm::Offset, llvm::peekThroughBitcasts(), llvm::APInt::popcount(), llvm::APInt::setBit(), llvm::ArrayRef< T >::size(), llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VZEXT_LOAD.

Referenced by combineConcatVectorOps(), combineToConsecutiveLoads(), and EltsFromConsecutiveLoads().

EmitAVX512Test()

static SDValue EmitAVX512Test ( SDValue  Op0, SDValue  Op1, ISD::CondCode  CC, const SDLoc &  dl, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, SDValue &  X86CC  )

static

Definition at line 24035 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, CC, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::isAllOnesConstant(), llvm::isNullConstant(), llvm::X86ISD::KORTEST, llvm::X86ISD::KTEST, LHS, llvm::ISD::OR, RHS, llvm::ISD::SETEQ, and llvm::ISD::SETNE.

EmitCmp()

static SDValue EmitCmp ( SDValue  Op0, SDValue  Op1, unsigned  X86CC, const SDLoc &  dl, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Emit nodes that will be selected as "cmp Op0,Op1", or something equivalent.

Definition at line 23022 of file X86ISelLowering.cpp.

References llvm::Add, llvm::X86ISD::ADD, assert(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::X86::COND_E, llvm::X86::COND_NE, EmitTest(), llvm::MachineFunction::getFunction(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::Function::hasMinSize(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), isX86CCSigned(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::X86::mayFoldLoad(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

emitLockedStackOp()

static SDValue emitLockedStackOp ( SelectionDAG &  DAG, const X86Subtarget &  Subtarget, SDValue  Chain, const SDLoc &  DL  )

static

Emit a locked operation on a stack location which does not change any memory location, but does involve a lock prefix.

Location is chosen to be a) very likely accessed only by a single thread to minimize cache traffic, and b) definitely dereferenceable. Returns the new Chain result.

Definition at line 31724 of file X86ISelLowering.cpp.

References DL, llvm::X86Subtarget::getFrameLowering(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), and llvm::X86FrameLowering::has128ByteRedZone().

Referenced by LowerATOMIC_FENCE(), LowerATOMIC_STORE(), and lowerAtomicArith().

EmitMaskedTruncSStore()

static SDValue EmitMaskedTruncSStore ( bool  SignedSat, SDValue  Chain, const SDLoc &  DL, SDValue  Val, SDValue  Ptr, SDValue  Mask, EVT  MemVT, MachineMemOperand *  MMO, SelectionDAG &  DAG  )

static

Emit Masked Truncating Store with signed or unsigned saturation.

Definition at line 27223 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getVTList(), Ptr, llvm::X86ISD::VMTRUNCSTORES, and llvm::X86ISD::VMTRUNCSTOREUS.

Referenced by LowerINTRINSIC_W_CHAIN().

emitOrXorXorTree()

template<typename F >

static SDValue emitOrXorXorTree ( SDValue  X, const SDLoc &  DL, SelectionDAG &  DAG, EVT  VecVT, EVT  CmpVT, bool  HasPT, F  SToV  )

static

Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp expansion.

Definition at line 22358 of file X86ISelLowering.cpp.

References A, llvm::ISD::AND, B, DL, emitOrXorXorTree(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm_unreachable, llvm::ISD::OR, llvm::ISD::SETEQ, llvm::ISD::SETNE, X, and llvm::ISD::XOR.

Referenced by combineVectorSizedSetCCEquality(), and emitOrXorXorTree().

EmitTest()

static SDValue EmitTest ( SDValue  Op, unsigned  X86CC, const SDLoc &  dl, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Emit nodes that will be selected as "test Op0,Op0", or something equivalent.

Definition at line 22912 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::ISD::AND, llvm::X86ISD::AND, llvm::X86ISD::CMP, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NO, llvm::X86::COND_O, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), hasNonFlagsUse(), isProfitableToUseFlagOp(), llvm_unreachable, llvm::ISD::MUL, llvm::ISD::OR, llvm::X86ISD::OR, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SHL, llvm::ISD::SSUBO, llvm::ISD::SUB, llvm::X86ISD::SUB, llvm::ISD::USUBO, llvm::ISD::XOR, and llvm::X86ISD::XOR.

Referenced by EmitCmp().

EmitTruncSStore()

static SDValue EmitTruncSStore ( bool  SignedSat, SDValue  Chain, const SDLoc &  DL, SDValue  Val, SDValue  Ptr, EVT  MemVT, MachineMemOperand *  MMO, SelectionDAG &  DAG  )

static

Emit Truncating Store with signed or unsigned saturation.

Definition at line 27212 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVTList(), Ptr, llvm::X86ISD::VTRUNCSTORES, and llvm::X86ISD::VTRUNCSTOREUS.

Referenced by combineStore(), and LowerINTRINSIC_W_CHAIN().

emitXBegin()

static MachineBasicBlock * emitXBegin ( MachineInstr &  MI, MachineBasicBlock *  MBB, const TargetInstrInfo *  TII  )

static

Utility function to emit xbegin specifying the start of an RTM region.

Definition at line 35196 of file X86ISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineBasicBlock::addLiveIn(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getBasicBlock(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, llvm::MachineFunction::insert(), isEFLAGSLiveAfter(), MBB, MI, MRI, llvm::MachineBasicBlock::splice(), TII, and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().

Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().

expandFP_TO_UINT_SSE()

static SDValue expandFP_TO_UINT_SSE ( MVT  VT, SDValue  Src, const SDLoc &  dl, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 21136 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::X86ISD::BLENDV, llvm::X86ISD::CVTTP2SI, llvm::ISD::FSUB, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::hasAVX2(), llvm::ISD::OR, and llvm::X86ISD::VSRAI.

Referenced by llvm::X86TargetLowering::ReplaceNodeResults().

ExpandHorizontalBinOp()

static SDValue ExpandHorizontalBinOp ( const SDValue &  V0, const SDValue &  V1, const SDLoc &  DL, SelectionDAG &  DAG, unsigned  X86Opcode, bool  Mode, bool  isUndefLO, bool  isUndefHI  )

static

Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.

This is a helper function of LowerToHorizontalOp(). This function expects two 256-bit vectors called V0 and V1. At first, each vector is split into two separate 128-bit vectors. Then, the resulting 128-bit vectors are used to implement two horizontal binary operations.

The kind of horizontal binary operation is defined by X86Opcode.

Mode specifies how the 128-bit parts of V0 and V1 are passed in input to the two new horizontal binop. When Mode is set, the first horizontal binop dag node would take as input the lower 128-bit of V0 and the upper 128-bit of V0. The second horizontal binop dag node would take as input the lower 128-bit of V1 and the upper 128-bit of V1. Example: HADD V0_LO, V0_HI HADD V1_LO, V1_HI

Otherwise, the first horizontal binop dag node takes as input the lower 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop dag node takes the upper 128-bit of V0 and the upper 128-bit of V1. Example: HADD V0_LO, V1_LO HADD V0_HI, V1_HI

If isUndefLO is set, then the algorithm propagates UNDEF to the lower 128-bits of the result. If isUndefHI is set, then UNDEF is propagated to the upper 128-bits of the result.

Definition at line 8033 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::is256BitVector(), and llvm::SDNode::isUndef().

Referenced by LowerToHorizontalOp().

expandIntrinsicWChainHelper()

static SDValue expandIntrinsicWChainHelper ( SDNode *  N, const SDLoc &  DL, SelectionDAG &  DAG, unsigned  TargetOpcode, unsigned  SrcReg, const X86Subtarget &  Subtarget, SmallVectorImpl< SDValue > &  Results  )

static

Handles the lowering of builtin intrinsics with chain that return their value into registers EDX:EAX.

If operand ScrReg is a valid register identifier, then operand 2 of N is copied to SrcReg. The assumption is that SrcReg is an implicit input to TargetOpcode. Returns a Glue value which can be used to add extra copy-from-reg if the expanded intrinsics implicitly defines extra registers (i.e. not just EDX:EAX).

Definition at line 27089 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_PAIR, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getMachineNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), N, llvm::ISD::OR, Results, and llvm::ISD::SHL.

Referenced by getReadTimeStampCounter(), LowerINTRINSIC_W_CHAIN(), and llvm::X86TargetLowering::ReplaceNodeResults().

ExtendToType()

static SDValue ExtendToType ( SDValue  InOp, MVT  NVT, SelectionDAG &  DAG, bool  FillWithZeroes = false  )

static

Widen a vector input to a vector of NVT.

The input vector must have the same element type as NVT.

Definition at line 32643 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::append(), assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDValue::isUndef(), llvm::SDNode::op_begin(), and llvm::SDNode::op_end().

Referenced by LowerMGATHER(), LowerMLOAD(), LowerMSCATTER(), and LowerMSTORE().

extract128BitVector()

static SDValue extract128BitVector ( SDValue  Vec, unsigned  IdxVal, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Generate a DAG to grab 128-bits from a vector > 128 bits.

This sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128 or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4 instructions or a simple subregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering EXTRACT_VECTOR_ELT operations easier.

Definition at line 4055 of file X86ISelLowering.cpp.

References assert(), extractSubVector(), llvm::SDValue::getValueType(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().

Referenced by canonicalizeShuffleMaskWithHorizOp(), combineArithReduction(), combineExtractWithShuffle(), combineTargetShuffle(), combineX86ShuffleChain(), ExpandHorizontalBinOp(), getTargetVShiftNode(), lowerAddSubToHorizontalOp(), lowerShuffleAsBroadcast(), LowerTruncateVecI1(), lowerV8I16Shuffle(), narrowExtractedVectorSelect(), llvm::X86TargetLowering::SimplifyDemandedBitsForTargetNode(), and vectorizeExtractedCast().

extract256BitVector()

static SDValue extract256BitVector ( SDValue  Vec, unsigned  IdxVal, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Generate a DAG to grab 256-bits from a 512-bit vector.

Definition at line 4063 of file X86ISelLowering.cpp.

References assert(), extractSubVector(), llvm::SDValue::getValueType(), and llvm::EVT::is512BitVector().

ExtractBitFromMaskVector()

static SDValue ExtractBitFromMaskVector ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Extract one bit from mask vector, like v16i1 or v8i1.

AVX-512 feature.

Definition at line 18245 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), Idx, llvm::X86ISD::KSHIFTR, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and widenMaskVector().

extractSubVector()

static SDValue extractSubVector ( SDValue  Vec, unsigned  IdxVal, SelectionDAG &  DAG, const SDLoc &  dl, unsigned  vectorWidth  )

static

Definition at line 4018 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::SDValue::isUndef(), and llvm::SDNode::ops().

findEltLoadSrc()

static bool findEltLoadSrc ( SDValue  Elt, LoadSDNode *&  Ld, int64_t &  ByteOffset  )

static

Definition at line 7012 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, findEltLoadSrc(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), Idx, llvm::ISD::isNON_EXTLoad(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SRL, and llvm::ISD::TRUNCATE.

Referenced by EltsFromConsecutiveLoads(), and findEltLoadSrc().

FindSingleBitChange()

static std::pair< Value *, BitTestKind > FindSingleBitChange ( Value *  V )

static

Definition at line 31261 of file X86ISelLowering.cpp.

References llvm::CallingConv::C, ConstantBit, I, llvm::isPowerOf2_64(), llvm::PatternMatch::m_AllOnes(), llvm::PatternMatch::m_c_And(), llvm::PatternMatch::m_Not(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_Sub(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), NotConstantBit, NotShiftBit, ShiftBit, and UndefBit.

FixupMMXIntrinsicTypes()

static SDValue FixupMMXIntrinsicTypes ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 59030 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), N, llvm::SDVTList::NumVTs, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SDVTList::VTs.

Referenced by combineINTRINSIC_VOID(), combineINTRINSIC_W_CHAIN(), and combineINTRINSIC_WO_CHAIN().

foldMaskedMerge()

static SDValue foldMaskedMerge ( SDNode *  Node, SelectionDAG &  DAG  )

static

Fold "masked merge" expressions like (m & x) | (~m & y) into the equivalent ((x ^ y) & m) ^ y) pattern.

This is typically a better representation for targets without a fused "and-not" operation. This function is intended to be called from a TargetLowering::PerformDAGCombine callback on ISD::OR nodes.

Definition at line 51434 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), DL, foldMaskedMergeImpl(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::hasOneUse(), and llvm::ISD::OR.

Referenced by combineOr().

foldMaskedMergeImpl()

static SDValue foldMaskedMergeImpl ( SDValue  And0_L, SDValue  And0_R, SDValue  And1_L, SDValue  And1_R, const SDLoc &  DL, SelectionDAG &  DAG  )

static

Definition at line 51408 of file X86ISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::ISD::FREEZE, llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::isBitwiseNot(), std::swap(), and llvm::ISD::XOR.

Referenced by foldMaskedMerge().

foldVectorXorShiftIntoCmp()

static SDValue foldVectorXorShiftIntoCmp ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1.

This should be called before type legalization because the pattern may not persist after that.

Definition at line 51925 of file X86ISelLowering.cpp.

References llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), llvm::isConstOrConstSplat(), llvm::EVT::isSimple(), N, llvm::ISD::SETGT, llvm::MVT::SimpleTy, and llvm::ISD::SRA.

foldXor1SetCC()

static SDValue foldXor1SetCC ( SDNode *  N, SelectionDAG &  DAG  )

static

Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)

Definition at line 54038 of file X86ISelLowering.cpp.

References DL, llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::isOneConstant(), LHS, N, llvm::X86ISD::SETCC, and llvm::ISD::XOR.

Referenced by combineXor().

foldXorTruncShiftIntoCmp()

static SDValue foldXorTruncShiftIntoCmp ( SDNode *  N, SelectionDAG &  DAG  )

static

Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1)

Definition at line 51869 of file X86ISelLowering.cpp.

References Cond, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::isOneConstant(), N, llvm::ISD::SETGT, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineXor().

getAltBitOpcode()

static unsigned getAltBitOpcode ( unsigned  Opcode )

static

Definition at line 44410 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::X86ISD::FAND, llvm::X86ISD::FANDN, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm_unreachable, llvm::ISD::OR, and llvm::ISD::XOR.

Referenced by adjustBitcastSrcVectorSSE1().

getAVX2GatherNode()

static SDValue getAVX2GatherNode ( unsigned  Opc, SDValue  Op, SelectionDAG &  DAG, SDValue  Src, SDValue  Mask, SDValue  Base, SDValue  Index, SDValue  ScaleOp, SDValue  Chain, const X86Subtarget &  Subtarget  )

static

Definition at line 26958 of file X86ISelLowering.cpp.

References llvm::sampleprof::Base, llvm::CallingConv::C, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::ISD::isBuildVectorAllOnes(), and llvm::X86ISD::MGATHER.

Referenced by LowerINTRINSIC_W_CHAIN().

getAVX512Node()

static SDValue getAVX512Node ( unsigned  Opcode, const SDLoc &  DL, MVT  VT, ArrayRef< SDValue >  Ops, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 4365 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, DL, extractSubVector(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), llvm::peekThroughBitcasts(), Widen(), and widenSubVector().

Referenced by canonicalizeBitSelect(), and LowerFunnelShift().

getAVX512TruncNode()

static SDValue getAVX512TruncNode ( const SDLoc &  DL, MVT  DstVT, SDValue  Src, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, bool  ZeroUppers  )

static

Definition at line 10297 of file X86ISelLowering.cpp.

References DL, extractSubVector(), getAVX512TruncNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::TRUNCATE, llvm::X86ISD::VTRUNC, and widenSubVector().

Referenced by getAVX512TruncNode(), lowerShuffleAsVTRUNC(), and lowerShuffleWithVPMOV().

getBitSelect()

static SDValue getBitSelect ( const SDLoc &  DL, MVT  VT, SDValue  LHS, SDValue  RHS, SDValue  Mask, SelectionDAG &  DAG  )

static

Definition at line 4643 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, DL, llvm::SelectionDAG::getNode(), LHS, llvm::ISD::OR, and RHS.

Referenced by lowerShuffleAsBitBlend().

getBMIMatchingOp()

static SDValue getBMIMatchingOp ( unsigned  Opc, SelectionDAG &  DAG, SDValue  OpMustEq, SDValue  Op, unsigned  Depth  )

static

Definition at line 50679 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::Depth, DL, getBMIMatchingOp(), llvm::SelectionDAG::getNode(), llvm::isAllOnesConstant(), llvm::isNullConstant(), llvm::isOneConstant(), and llvm::ISD::SUB.

Referenced by combineBMILogicOp(), and getBMIMatchingOp().

getBROADCAST_LOAD()

static SDValue getBROADCAST_LOAD ( unsigned  Opcode, const SDLoc &  DL, EVT  VT, EVT  MemVT, MemSDNode *  Mem, unsigned  Offset, SelectionDAG &  DAG  )

static

Definition at line 6535 of file X86ISelLowering.cpp.

References assert(), DL, llvm::MemSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::EVT::getStoreSize(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MemSDNode::isNonTemporal(), llvm::MemSDNode::isSimple(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::Offset, Ptr, llvm::MemSDNode::readMem(), llvm::X86ISD::SUBV_BROADCAST_LOAD, and llvm::X86ISD::VBROADCAST_LOAD.

Referenced by combineConcatVectorOps(), combineINSERT_SUBVECTOR(), lowerV2X128Shuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

getBT()

static SDValue getBT ( SDValue  Src, SDValue  BitNo, const SDLoc &  DL, SelectionDAG &  DAG  )

static

Helper for attempting to create a X86ISD::BT node.

Definition at line 22297 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::X86ISD::BT, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SelectionDAG::MaskedValueIsZero(), and llvm::ISD::TRUNCATE.

Referenced by combineAnd(), combineCarryThroughADD(), and LowerAndToBT().

getConstantVector() [1/2]

static Constant * getConstantVector ( MVT  VT, ArrayRef< APInt >  Bits, const APInt &  Undefs, LLVMContext &  C  )

static

Definition at line 7358 of file X86ISelLowering.cpp.

References assert(), llvm::CallingConv::C, llvm::ConstantVector::get(), llvm::UndefValue::get(), llvm::Constant::getIntegerValue(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::EVT::getTypeForEVT(), I, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEhalf(), llvm::APFloatBase::IEEEsingle(), llvm::MVT::isFloatingPoint(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by combineConcatVectorOps(), and lowerBuildVectorAsBroadcast().

getConstantVector() [2/2]

static Constant * getConstantVector ( MVT  VT, const APInt &  SplatValue, unsigned  SplatBitSize, LLVMContext &  C  )

static

Definition at line 7383 of file X86ISelLowering.cpp.

References assert(), llvm::CallingConv::C, llvm::APInt::extractBits(), llvm::ConstantVector::get(), llvm::Constant::getIntegerValue(), llvm::Type::getIntNTy(), llvm::MVT::getScalarSizeInBits(), I, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEhalf(), llvm::APFloatBase::IEEEsingle(), llvm::MVT::isFloatingPoint(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

getConstVector() [1/3]

static SDValue getConstVector ( ArrayRef< APInt >  Bits, const APInt &  Undefs, MVT  VT, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Definition at line 3920 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::append(), assert(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEsingle(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

getConstVector() [2/3]

static SDValue getConstVector ( ArrayRef< APInt >  Bits, MVT  VT, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Definition at line 3961 of file X86ISelLowering.cpp.

References getConstVector(), and llvm::APInt::getZero().

getConstVector() [3/3]

static SDValue getConstVector ( ArrayRef< int >  Values, MVT  VT, SelectionDAG &  DAG, const SDLoc &  dl, bool  IsMask = false  )

static

Definition at line 3890 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by combineAndnp(), combineConcatVectorOps(), combineSelect(), combineTargetShuffle(), combineVectorCompare(), combineVectorPack(), combineVectorShiftImm(), combineVPMADD(), combineX86ShuffleChain(), combineX86ShufflesConstants(), getConstVector(), IsNOT(), LowerShift(), lowerShuffleAsElementInsertion(), lowerShuffleWithPERMV(), lowerV16F32Shuffle(), lowerV16I8Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().

getEXTEND_VECTOR_INREG()

static SDValue getEXTEND_VECTOR_INREG ( unsigned  Opcode, const SDLoc &  DL, EVT  VT, SDValue  In, SelectionDAG &  DAG  )

static

Definition at line 4617 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), DL, extractSubVector(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::EVT::isVector(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

Referenced by combineVectorPack(), lowerShuffleAsSpecificZeroOrAnyExtend(), and llvm::X86TargetLowering::ReplaceNodeResults().

getExtractedDemandedElts()

static APInt getExtractedDemandedElts ( SDNode *  N )

static

Definition at line 18291 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, getExtractedDemandedElts(), llvm::User::getOperand(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZero(), N, llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, llvm::APIntOps::ScaleBitMask(), llvm::APInt::setAllBits(), and llvm::APInt::setBit().

Referenced by getExtractedDemandedElts().

getFauxShuffleMask()

static bool getFauxShuffleMask ( SDValue  N, const APInt &  DemandedElts, SmallVectorImpl< int > &  Mask, SmallVectorImpl< SDValue > &  Ops, const SelectionDAG &  DAG, unsigned  Depth, bool  ResolveKnownElts  )

static

Definition at line 5944 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::ISD::ANY_EXTEND, llvm::ISD::ANY_EXTEND_VECTOR_INREG, llvm::any_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), llvm::ISD::BITCAST, llvm::X86ISD::BLENDV, llvm::SmallVectorImpl< T >::clear(), llvm::SelectionDAG::ComputeNumSignBits(), Cond, createPackShuffleMask(), createShuffleMaskFromVSELECT(), llvm::DecodeZeroExtendMask(), llvm::Depth, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::getAllOnes(), llvm::APInt::getBitWidth(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::EVT::getFixedSizeInBits(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPackDemandedElts(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetShuffleInputs(), llvm::SelectionDAG::getValidShiftAmount(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isByteSized(), isInRange(), llvm::isNullConstant(), llvm::EVT::isSimple(), llvm::SDValue::isUndef(), isUndefOrInRange(), llvm::EVT::isVector(), llvm::APInt::isZero(), llvm::X86::isZeroNode(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::narrowShuffleMaskElts(), llvm::Offset, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::X86ISD::VBROADCAST, llvm::ISD::VECTOR_SHUFFLE, llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::ISD::VSELECT, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, llvm::X86ISD::VTRUNC, llvm::ISD::ZERO_EXTEND, llvm::ISD::ZERO_EXTEND_VECTOR_INREG, and llvm::APInt::zextOrTrunc().

Referenced by getTargetShuffleInputs().

getFlagsOfCmpZeroFori1()

static SDValue getFlagsOfCmpZeroFori1 ( SelectionDAG &  DAG, const SDLoc &  DL, SDValue  Mask  )

static

Definition at line 33048 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), and llvm::X86ISD::SUB.

Referenced by llvm::X86TargetLowering::visitMaskedLoad(), and llvm::X86TargetLowering::visitMaskedStore().

getGatherNode()

static SDValue getGatherNode ( SDValue  Op, SelectionDAG &  DAG, SDValue  Src, SDValue  Mask, SDValue  Base, SDValue  Index, SDValue  ScaleOp, SDValue  Chain, const X86Subtarget &  Subtarget  )

static

Definition at line 26990 of file X86ISelLowering.cpp.

References llvm::sampleprof::Base, llvm::CallingConv::C, llvm::SelectionDAG::getDataLayout(), getMaskNode(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::ISD::isBuildVectorAllOnes(), and llvm::X86ISD::MGATHER.

Referenced by LowerINTRINSIC_W_CHAIN().

getGFNICtrlImm()

uint64_t getGFNICtrlImm	(	unsigned	Opcode,
		unsigned	Amt = 0
	)

Definition at line 28441 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITREVERSE, getGFNICtrlImm(), llvm_unreachable, llvm::ISD::ROTL, llvm::ISD::ROTR, llvm::ISD::SHL, llvm::ISD::SRA, and llvm::ISD::SRL.

Referenced by getGFNICtrlImm(), and getGFNICtrlMask().

getGFNICtrlMask()

SDValue getGFNICtrlMask	(	unsigned	Opcode,
		SelectionDAG &	DAG,
		const SDLoc &	DL,
		MVT	VT,
		unsigned	Amt = 0
	)

Definition at line 28464 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getGFNICtrlImm(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by LowerBITREVERSE(), LowerRotate(), and LowerShiftByScalarImmediate().

getHalfShuffleMask()

static bool getHalfShuffleMask ( ArrayRef< int >  Mask, MutableArrayRef< int >  HalfMask, int &  HalfIdx1, int &  HalfIdx2  )

static

If the input shuffle mask results in a vector that is undefined in all upper or lower half elements and that mask accesses only 2 halves of the shuffle's operands, return true.

A mask of half the width with mask indexes adjusted to access the extracted halves of the original shuffle operands is returned in HalfMask. HalfIdx1 and HalfIdx2 return whether the upper or lower half of each input operand is accessed.

Definition at line 15467 of file X86ISelLowering.cpp.

References assert(), isUndefLowerHalf(), isUndefUpperHalf(), and llvm::ArrayRef< T >::size().

Referenced by lowerShuffleWithUndefHalf(), and narrowShuffle().

getHopForBuildVector()

static SDValue getHopForBuildVector ( const BuildVectorSDNode *  BV, const SDLoc &  DL, SelectionDAG &  DAG, unsigned  HOpcode, SDValue  V0, SDValue  V1  )

static

Definition at line 8358 of file X86ISelLowering.cpp.

References llvm::APInt::clearBit(), DL, extractSubVector(), llvm::APInt::getAllOnes(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorNumElements(), insertSubVector(), llvm::MVT::is256BitVector(), llvm::SDValue::isUndef(), and llvm::APInt::lshr().

Referenced by LowerToHorizontalOp().

getHorizDemandedElts()

static void getHorizDemandedElts ( EVT  VT, const APInt &  DemandedElts, APInt &  DemandedLHS, APInt &  DemandedRHS  )

static

Definition at line 5350 of file X86ISelLowering.cpp.

References llvm::getHorizDemandedEltsForFirstOperand(), and llvm::EVT::getSizeInBits().

Referenced by llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

getIndexFromUnindexedLoad()

static SDValue getIndexFromUnindexedLoad ( LoadSDNode *  Ld )

static

Definition at line 50468 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::sampleprof::Base, llvm::LoadSDNode::getBasePtr(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::LSBaseSDNode::isIndexed(), and llvm::ISD::SHL.

Referenced by combineAndLoadToBZHI().

getIndirectThunkSymbol()

static const char * getIndirectThunkSymbol ( const X86Subtarget &  Subtarget, unsigned  Reg  )

static

Definition at line 36314 of file X86ISelLowering.cpp.

References assert(), and llvm_unreachable.

getInstrStrFromOpNo()

static StringRef getInstrStrFromOpNo ( const SmallVectorImpl< StringRef > &  AsmStrs, unsigned  OpNo  )

static

Definition at line 32994 of file X86ISelLowering.cpp.

References assert(), I, llvm::StringRef::npos, and llvm::toString().

Referenced by llvm::X86TargetLowering::isInlineAsmTargetBranch().

getInvertedVectorForFMA()

static SDValue getInvertedVectorForFMA ( SDValue  V, SelectionDAG &  DAG  )

static

Definition at line 55063 of file X86ISelLowering.cpp.

References llvm::any_of(), assert(), llvm::ISD::BUILD_VECTOR, llvm::ISD::FMA, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::STRICT_FMA.

Referenced by combineFMA().

getMaskNode()

static SDValue getMaskNode ( SDValue  Mask, MVT  MaskVT, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics.

Definition at line 25754 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::bitsLE(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorVT(), llvm::Hi, llvm::isAllOnesConstant(), llvm::X86::isZeroNode(), llvm::Lo, and llvm::SelectionDAG::SplitScalar().

Referenced by getGatherNode(), getPrefetchNode(), getScatterNode(), getVectorMaskingNode(), LowerINTRINSIC_W_CHAIN(), and lowerShuffleWithEXPAND().

getNullFPConstForNullVal()

static SDValue getNullFPConstForNullVal ( SDValue  V, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value.

In the case of a vector, the returned constant will not contain undefined elements even if the input parameter does. This makes it suitable to be used as a replacement operand with operations (eg, bitwise-and) where an undef should not propagate.

Definition at line 54278 of file X86ISelLowering.cpp.

References getZeroVector(), and isNullFPScalarOrVectorConst().

Referenced by combineFAnd(), and combineFAndn().

getOnesVector()

static SDValue getOnesVector ( EVT  VT, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Returns a vector of specified type with all bits set.

Always build ones vectors as <4 x i32>, <8 x i32> or <16 x i32>. Then bitcast to their original type, ensuring they get CSE'd.

Definition at line 4609 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().

Referenced by combineEXTRACT_SUBVECTOR(), combineX86ShufflesRecursively(), and materializeVectorConstant().

getOneTrueElt()

static int getOneTrueElt ( SDValue  V )

static

If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element.

Otherwise, return -1.

Definition at line 52331 of file X86ISelLowering.cpp.

References llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), and llvm::EVT::getVectorNumElements().

Referenced by getParamsForOneTrueMaskedElt().

getOpcodeForIndirectThunk()

static unsigned getOpcodeForIndirectThunk ( unsigned  RPOpc )

static

Definition at line 36300 of file X86ISelLowering.cpp.

References llvm_unreachable.

getPack()

static SDValue getPack ( SelectionDAG &  DAG, const X86Subtarget &  Subtarget, const SDLoc &  dl, MVT  VT, SDValue  LHS, SDValue  RHS, bool  PackHiHalf = false  )

static

Returns a node that packs the LHS + RHS nodes together at half width.

May return X86ISD::PACKSS/PACKUS, packing the top/bottom half. TODO: Add subvector splitting if/when we have a need for it.

Definition at line 4721 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::KnownBits::countMaxActiveBits(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), I, LHS, llvm::Offset, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.

Referenced by convertShiftLeftToScale(), LowerFunnelShift(), LowerMUL(), LowerRotate(), and LowervXi8MulWithUNPCK().

getPackDemandedElts()

static void getPackDemandedElts ( EVT  VT, const APInt &  DemandedElts, APInt &  DemandedLHS, APInt &  DemandedRHS  )

static

Definition at line 5325 of file X86ISelLowering.cpp.

References llvm::APInt::getBitWidth(), llvm::EVT::getSizeInBits(), llvm::APInt::getZero(), and llvm::APInt::setBit().

Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode(), llvm::X86TargetLowering::ComputeNumSignBitsForTargetNode(), getFauxShuffleMask(), llvm::X86TargetLowering::SimplifyDemandedBitsForTargetNode(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

getParamsForOneTrueMaskedElt()

static bool getParamsForOneTrueMaskedElt ( MaskedLoadStoreSDNode *  MaskedOp, SelectionDAG &  DAG, SDValue &  Addr, SDValue &  Index, Align &  Alignment, unsigned &  Offset  )

static

Given a masked memory load/store operation, return true if it has one mask bit set.

If it has one mask bit set, then also return the memory address of the scalar element to load/store, the vector index to insert/extract that scalar element, and the alignment for the scalar memory access.

Definition at line 52366 of file X86ISelLowering.cpp.

References Addr, llvm::commonAlignment(), llvm::MemSDNode::getBasePtr(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemoryVT(), getOneTrueElt(), llvm::MemSDNode::getOriginalAlign(), llvm::EVT::getStoreSize(), llvm::EVT::getVectorElementType(), and llvm::Offset.

Referenced by reduceMaskedLoadToScalarLoad(), and reduceMaskedStoreToScalarStore().

getPMOVMSKB()

static SDValue getPMOVMSKB ( const SDLoc &  DL, SDValue  V, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 31847 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPMOVMSKB(), llvm::X86Subtarget::hasInt256(), llvm::Hi, llvm::Lo, llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::ISD::SHL, llvm::SelectionDAG::SplitVector(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineBitcastvxi1(), combinePredicateReduction(), combinePTESTCC(), getPMOVMSKB(), and LowerBITCAST().

getPrefetchNode()

static SDValue getPrefetchNode ( unsigned  Opc, SDValue  Op, SelectionDAG &  DAG, SDValue  Mask, SDValue  Base, SDValue  Index, SDValue  ScaleOp, SDValue  Chain, const X86Subtarget &  Subtarget  )

static

Definition at line 27059 of file X86ISelLowering.cpp.

References llvm::sampleprof::Base, llvm::CallingConv::C, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), getMaskNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), and llvm::MVT::getVectorVT().

Referenced by LowerINTRINSIC_W_CHAIN().

getPSHUFShuffleMask()

static SmallVector< int, 4 > getPSHUFShuffleMask ( SDValue  N )

static

Get the PSHUF-style mask from PSHUF node.

This is a very minor wrapper around getTargetShuffleMask to easy forming v4 PSHUF-style masks that can be reused with such instructions.

Definition at line 41045 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), getTargetShuffleMask(), llvm_unreachable, N, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, and llvm::X86ISD::PSHUFLW.

Referenced by combineRedundantDWordShuffle(), and combineTargetShuffle().

getReadTimeStampCounter()

static void getReadTimeStampCounter ( SDNode *  N, const SDLoc &  DL, unsigned  Opcode, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, SmallVectorImpl< SDValue > &  Results  )

static

Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp).

This function is also used to custom lower READCYCLECOUNTER nodes.

Definition at line 27144 of file X86ISelLowering.cpp.

References DL, expandIntrinsicWChainHelper(), llvm::SelectionDAG::getCopyFromReg(), llvm::SDValue::getValue(), N, and Results.

Referenced by LowerINTRINSIC_W_CHAIN(), LowerREADCYCLECOUNTER(), and llvm::X86TargetLowering::ReplaceNodeResults().

getScalarMaskingNode()

static SDValue getScalarMaskingNode ( SDValue  Op, SDValue  Mask, SDValue  PreservedSrc, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Creates an SDNode for a predicated scalar operation.

Returns

(X86vselect Mask, Op, PreservedSrc). The mask is coming as MVT::i8 and it should be transformed to MVT::v1i1 while lowering masking intrinsics. The main difference between ScalarMaskingNode and VectorMaskingNode is using "X86select" instead of "vselect". We just can't create the "vselect" node for a scalar instruction.

Definition at line 25814 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::X86ISD::FSETCCM, llvm::X86ISD::FSETCCM_SAE, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getZeroVector(), llvm::SDValue::isUndef(), llvm::X86ISD::SELECTS, and llvm::X86ISD::VFPCLASSS.

getScalarValueForVectorElement()

static SDValue getScalarValueForVectorElement ( SDValue  V, int  Idx, SelectionDAG &  DAG  )

static

Try to get a scalar value for a specific element of a vector.

Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.

Definition at line 12376 of file X86ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), Idx, llvm::MVT::isVector(), llvm::peekThroughBitcasts(), and llvm::ISD::SCALAR_TO_VECTOR.

Referenced by lowerShuffleAsElementInsertion(), and lowerV2F64Shuffle().

getScatterNode()

static SDValue getScatterNode ( unsigned  Opc, SDValue  Op, SelectionDAG &  DAG, SDValue  Src, SDValue  Mask, SDValue  Base, SDValue  Index, SDValue  ScaleOp, SDValue  Chain, const X86Subtarget &  Subtarget  )

static

Definition at line 27028 of file X86ISelLowering.cpp.

References llvm::sampleprof::Base, llvm::CallingConv::C, llvm::SelectionDAG::getDataLayout(), getMaskNode(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), and llvm::X86ISD::MSCATTER.

Referenced by LowerINTRINSIC_W_CHAIN().

getSEHRegistrationNodeSize()

static int getSEHRegistrationNodeSize ( const Function *  Fn )

static

Definition at line 25840 of file X86ISelLowering.cpp.

References llvm::classifyEHPersonality(), llvm::Function::getPersonalityFn(), llvm::Function::hasPersonalityFn(), llvm::MSVC_CXX, llvm::MSVC_X86SEH, and llvm::report_fatal_error().

Referenced by recoverFramePointer().

getSETCC()

static SDValue getSETCC ( X86::CondCode  Cond, SDValue  EFLAGS, const SDLoc &  dl, SelectionDAG &  DAG  )

static

Helper for creating a X86ISD::SETCC node.

Definition at line 22338 of file X86ISelLowering.cpp.

References Cond, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), and llvm::X86ISD::SETCC.

getShuffleHalfVectors()

static SDValue getShuffleHalfVectors ( const SDLoc &  DL, SDValue  V1, SDValue  V2, ArrayRef< int >  HalfMask, int  HalfIdx1, int  HalfIdx2, bool  UndefLower, SelectionDAG &  DAG, bool  UseConcat = false  )

static

Given the output values from getHalfShuffleMask(), create a half width shuffle of extracted vectors followed by an insert back to full width.

Definition at line 15518 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::isSimple(), llvm::Offset, and std::swap().

Referenced by lowerShuffleWithUndefHalf(), and narrowShuffle().

getShuffleScalarElt()

static SDValue getShuffleScalarElt ( SDValue  Op, unsigned  Index, SelectionDAG &  DAG, unsigned  Depth  )

static

Returns the scalar element that will make up the i'th element of the result of the vector shuffle.

Definition at line 6560 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::Depth, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), getShuffleScalarElt(), llvm::EVT::getSimpleVT(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::MVT::isInteger(), isTargetShuffle(), llvm::EVT::isVector(), llvm::SelectionDAG::MaxRecursionDepth, llvm::ISD::SCALAR_TO_VECTOR, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineToConsecutiveLoads(), and getShuffleScalarElt().

getShuffleVectorZeroOrUndef()

static SDValue getShuffleVectorZeroOrUndef ( SDValue  V2, int  Idx, bool  IsZero, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Return a vector_shuffle of the specified vector of zero or undef vector.

This produces a shuffle where the low element of V2 is swizzled into the zero/undef vector, landing at element Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).

Definition at line 4788 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), and Idx.

Referenced by LowerUINT_TO_FP_i32().

getSHUFPDImm()

static unsigned getSHUFPDImm ( ArrayRef< int >  Mask )

static

Definition at line 9968 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), llvm::count_if(), llvm::find_if(), and I.

Referenced by getSHUFPDImmForMask(), and matchShuffleWithSHUFPD().

getSHUFPDImmForMask()

static SDValue getSHUFPDImmForMask ( ArrayRef< int >  Mask, const SDLoc &  DL, SelectionDAG &  DAG  )

static

Definition at line 9998 of file X86ISelLowering.cpp.

References DL, getSHUFPDImm(), and llvm::SelectionDAG::getTargetConstant().

Referenced by lowerShuffleAsLanePermuteAndSHUFP().

getSplitVectorSrc()

static SDValue getSplitVectorSrc ( SDValue  LHS, SDValue  RHS, bool  AllowCommute  )

static

Definition at line 3997 of file X86ISelLowering.cpp.

References llvm::ISD::EXTRACT_SUBVECTOR, LHS, and RHS.

Referenced by combineHorizOpWithShuffle(), combinePTESTCC(), and combineSetCCMOVMSK().

getSUBriOpcode()

static unsigned getSUBriOpcode ( bool  IsLP64 )

static

Definition at line 35967 of file X86ISelLowering.cpp.

getTargetConstantBitsFromNode()

static bool getTargetConstantBitsFromNode ( SDValue  Op, unsigned  EltSizeInBits, APInt &  UndefElts, SmallVectorImpl< APInt > &  EltBits, bool  AllowWholeUndefs = true, bool  AllowPartialUndefs = false  )

static

Definition at line 4836 of file X86ISelLowering.cpp.

References llvm::any_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::CallingConv::C, llvm::SmallVectorBase< Size_T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::extractBits(), llvm::Constant::getAggregateElement(), llvm::APInt::getAllOnes(), llvm::APInt::getBitWidth(), getBitWidth(), llvm::APInt::getBoolValue(), llvm::Type::getPrimitiveSizeInBits(), llvm::EVT::getScalarSizeInBits(), llvm::Type::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::APInt::getSplat(), getTargetConstantBitsFromNode(), getTargetConstantFromBasePtr(), getTargetConstantFromNode(), llvm::Value::getType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::APInt::insertBits(), llvm::APInt::isAllOnes(), isAnyInRange(), llvm::Type::isDoubleTy(), llvm::Type::isFloatTy(), llvm::Type::isHalfTy(), llvm::Type::isIntegerTy(), llvm::Type::isVectorTy(), llvm::peekThroughBitcasts(), Ptr, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SCALAR_TO_VECTOR, llvm::APInt::setBit(), llvm::APInt::setBits(), llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::X86ISD::VBROADCAST_LOAD, and llvm::X86ISD::VZEXT_MOVL.

Referenced by canonicalizeBitSelect(), combineAnd(), combineAndnp(), combineBitcast(), combineConcatVectorOps(), combineConstantPoolLoads(), combineExtractVectorElt(), combineMOVMSK(), combineOr(), combineVectorCompare(), combineVectorPack(), combineVectorShiftImm(), combineVectorShiftVar(), combineVPMADD(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), llvm::X86TargetLowering::computeKnownBitsForTargetNode(), convertShiftLeftToScale(), createShuffleMaskFromVSELECT(), getFauxShuffleMask(), getTargetConstantBitsFromNode(), getTargetShuffleAndZeroables(), getTargetShuffleMaskIndices(), isConstantPowerOf2(), llvm::X86::isConstantSplat(), isFNEG(), IsNOT(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

getTargetConstantFromBasePtr()

static const Constant * getTargetConstantFromBasePtr ( SDValue  Ptr )

static

Definition at line 4811 of file X86ISelLowering.cpp.

References llvm::ConstantPoolSDNode::getConstVal(), llvm::ConstantPoolSDNode::getOffset(), getTargetConstantPoolFromBasePtr(), llvm::ConstantPoolSDNode::isMachineConstantPoolEntry(), and Ptr.

Referenced by combineConstantPoolLoads(), getTargetConstantBitsFromNode(), and getTargetConstantFromNode().

getTargetConstantFromNode() [1/2]

static const Constant * getTargetConstantFromNode ( LoadSDNode *  Load )

static

Definition at line 4818 of file X86ISelLowering.cpp.

References getTargetConstantFromBasePtr(), and llvm::ISD::isNormalLoad().

Referenced by canonicalizeShuffleWithOp(), combineAnd(), combineConcatVectorOps(), getTargetConstantBitsFromNode(), llvm::X86TargetLowering::getTargetConstantFromLoad(), getTargetConstantFromNode(), lowerShuffleAsElementInsertion(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle().

getTargetConstantFromNode() [2/2]

static const Constant * getTargetConstantFromNode ( SDValue  Op )

static

Definition at line 4824 of file X86ISelLowering.cpp.

References getTargetConstantFromNode(), and llvm::peekThroughBitcasts().

getTargetConstantPoolFromBasePtr()

static ConstantPoolSDNode * getTargetConstantPoolFromBasePtr ( SDValue  Ptr )

static

Definition at line 4803 of file X86ISelLowering.cpp.

References Ptr, llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.

Referenced by getTargetConstantFromBasePtr().

getTargetShuffleAndZeroables()

static bool getTargetShuffleAndZeroables ( SDValue  N, SmallVectorImpl< int > &  Mask, SmallVectorImpl< SDValue > &  Ops, APInt &  KnownUndef, APInt &  KnownZero  )

static

Decode a target shuffle mask and inputs and see if any values are known to be undef or zero from their inputs.

Returns true if the target shuffle mask was decoded. FIXME: Merge this with computeZeroableShuffleElements?

Definition at line 5770 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetShuffleMask(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZero(), Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::isFloatingPoint(), isTargetShuffle(), llvm::SDValue::isUndef(), isUndefOrZero(), llvm::X86::isZeroNode(), N, llvm::peekThroughBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::APInt::setBit(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineTargetShuffle(), and getTargetShuffleInputs().

getTargetShuffleInputs() [1/3]

static bool getTargetShuffleInputs ( SDValue  Op, const APInt &  DemandedElts, SmallVectorImpl< SDValue > &  Inputs, SmallVectorImpl< int > &  Mask, APInt &  KnownUndef, APInt &  KnownZero, const SelectionDAG &  DAG, unsigned  Depth, bool  ResolveKnownElts  )

static

Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs and then sets the SM_SentinelUndef and SM_SentinelZero values.

Returns true if the target shuffle mask was decoded.

Definition at line 6484 of file X86ISelLowering.cpp.

References llvm::Depth, getFauxShuffleMask(), getTargetShuffleAndZeroables(), llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::SelectionDAG::MaxRecursionDepth, resolveTargetShuffleFromZeroables(), and resolveZeroablesFromTargetShuffle().

getTargetShuffleInputs() [2/3]

static bool getTargetShuffleInputs ( SDValue  Op, const APInt &  DemandedElts, SmallVectorImpl< SDValue > &  Inputs, SmallVectorImpl< int > &  Mask, const SelectionDAG &  DAG, unsigned  Depth, bool  ResolveKnownElts  )

static

Definition at line 6510 of file X86ISelLowering.cpp.

References llvm::Depth, and getTargetShuffleInputs().

Referenced by combineEXTRACT_SUBVECTOR(), combineExtractWithShuffle(), combineHorizOpWithShuffle(), combineSetCCMOVMSK(), combineTargetShuffle(), combineX86ShufflesRecursively(), getFauxShuffleMask(), getTargetShuffleInputs(), isHorizontalBinOp(), llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(), and llvm::X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode().

getTargetShuffleInputs() [3/3]

static bool getTargetShuffleInputs ( SDValue  Op, SmallVectorImpl< SDValue > &  Inputs, SmallVectorImpl< int > &  Mask, const SelectionDAG &  DAG, unsigned  Depth = 0, bool  ResolveKnownElts = true  )

static

Definition at line 6520 of file X86ISelLowering.cpp.

References llvm::Depth, llvm::APInt::getAllOnes(), getTargetShuffleInputs(), llvm::EVT::isSimple(), and llvm::EVT::isVector().

getTargetShuffleMask() [1/2]

static bool getTargetShuffleMask ( SDValue  N, bool  AllowSentinelZero, SmallVectorImpl< SDValue > &  Ops, SmallVectorImpl< int > &  Mask  )

static

Definition at line 5672 of file X86ISelLowering.cpp.

References getTargetShuffleMask(), and N.

getTargetShuffleMask() [2/2]

static bool getTargetShuffleMask ( SDValue  N, bool  AllowSentinelZero, SmallVectorImpl< SDValue > &  Ops, SmallVectorImpl< int > &  Mask, bool &  IsUnary  )

static

Calculates the shuffle mask corresponding to the target-specific opcode.

If the mask could be calculated, returns it in Mask, returns the shuffle operands in Ops, and returns true. Sets IsUnary to true if only one source is used. Note that this will set IsUnary for shuffles which use a single input multiple times, and in those cases it will adjust the mask to only have indices within that single input. It is an error to call this with non-empty Mask/Ops vectors.

Definition at line 5365 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::BLENDI, llvm::DecodeBLENDMask(), llvm::DecodeEXTRQIMask(), llvm::DecodeINSERTPSMask(), llvm::DecodeINSERTQIMask(), llvm::DecodeMOVDDUPMask(), llvm::DecodeMOVHLPSMask(), llvm::DecodeMOVLHPSMask(), llvm::DecodeMOVSHDUPMask(), llvm::DecodeMOVSLDUPMask(), llvm::DecodePALIGNRMask(), llvm::DecodePSHUFBMask(), llvm::DecodePSHUFHWMask(), llvm::DecodePSHUFLWMask(), llvm::DecodePSHUFMask(), llvm::DecodePSLLDQMask(), llvm::DecodePSRLDQMask(), llvm::DecodeScalarMoveMask(), llvm::DecodeSHUFPMask(), llvm::DecodeUNPCKHMask(), llvm::DecodeUNPCKLMask(), llvm::DecodeVALIGNMask(), llvm::DecodeVectorBroadcast(), llvm::DecodeVPERM2X128Mask(), llvm::DecodeVPERMIL2PMask(), llvm::DecodeVPERMILPMask(), llvm::DecodeVPERMMask(), llvm::DecodeVPERMV3Mask(), llvm::DecodeVPERMVMask(), llvm::DecodeVPPERMMask(), llvm::decodeVSHUF64x2FamilyMask(), llvm::DecodeZeroMoveLowMask(), llvm::SmallVectorBase< Size_T >::empty(), llvm::X86ISD::EXTRQI, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), getTargetShuffleMaskIndices(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::INSERTPS, llvm::X86ISD::INSERTQI, isAnyZero(), isTargetShuffle(), llvm_unreachable, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, N, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VALIGN, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VZEXT_MOVL.

Referenced by canonicalizeShuffleWithOp(), combineBlendOfPermutes(), combineConcatVectorOps(), combineSelect(), combineTargetShuffle(), llvm::X86TargetLowering::computeKnownBitsForTargetNode(), llvm::X86TargetLowering::ComputeNumSignBitsForTargetNode(), getPSHUFShuffleMask(), getShuffleScalarElt(), getTargetShuffleAndZeroables(), getTargetShuffleMask(), and llvm::X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode().

getTargetShuffleMaskIndices()

static bool getTargetShuffleMaskIndices ( SDValue  MaskNode, unsigned  MaskEltSizeInBits, SmallVectorImpl< uint64_t > &  RawMask, APInt &  UndefElts  )

static

Definition at line 5194 of file X86ISelLowering.cpp.

References getTargetConstantBitsFromNode(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by getTargetShuffleMask().

getTargetVShiftByConstNode()

static SDValue getTargetVShiftByConstNode ( unsigned  Opc, const SDLoc &  dl, MVT  VT, SDValue  SrcOp, uint64_t  ShiftAmt, SelectionDAG &  DAG  )

static

Handle vector element shifts where the shift amount is a constant.

Takes immediate version of shift as input.

Definition at line 25605 of file X86ISelLowering.cpp.

References assert(), llvm::CallingConv::C, llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorElementType(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm_unreachable, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.

Referenced by combineAndMaskToShift(), combineMOVMSK(), combineScalarToVector(), combineVectorShiftVar(), LowerFunnelShift(), LowerMUL(), LowerMULH(), LowerMULO(), LowerRotate(), LowerShift(), LowerShiftByScalarImmediate(), and LowerShiftByScalarVariable().

getTargetVShiftNode()

static SDValue getTargetVShiftNode ( unsigned  Opc, const SDLoc &  dl, MVT  VT, SDValue  SrcOp, SDValue  ShAmt, int  ShAmtIdx, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle vector element shifts by a splat shift amount.

Definition at line 25657 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BUILD_VECTOR, extract128BitVector(), llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is128BitVector(), llvm::EVT::isSimple(), llvm::MVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, llvm::X86ISD::VZEXT_MOVL, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by LowerFunnelShift(), LowerRotate(), and LowerShiftByScalarVariable().

getTargetVShiftUniformOpcode()

static unsigned getTargetVShiftUniformOpcode ( unsigned  Opc, bool  IsVariable  )

static

Definition at line 25585 of file X86ISelLowering.cpp.

References llvm_unreachable, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, and llvm::X86ISD::VSRLI.

Referenced by combineScalarToVector(), combineVectorShiftVar(), getTargetVShiftNode(), LowerShift(), LowerShiftByScalarImmediate(), and LowerShiftByScalarVariable().

GetTLSADDR()

static SDValue GetTLSADDR ( SelectionDAG &  DAG, GlobalAddressSDNode *  GA, const EVT  PtrVT, unsigned  ReturnReg, unsigned char  OperandFlags, bool  LoadGlobalBaseReg = false, bool  LocalDynamic = false  )

static

Definition at line 18966 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::ISD::CALLSEQ_END, llvm::ISD::CopyFromReg, llvm::X86AS::FS, llvm::PointerType::get(), llvm::SelectionDAG::getCALLSEQ_END(), llvm::SelectionDAG::getCALLSEQ_START(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getEntryNode(), llvm::MachineFunction::getFrameInfo(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SDNode::getGluedUser(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::GlobalAddressSDNode::getOffset(), llvm::SDNode::getOpcode(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86ISD::GlobalBaseReg, llvm::X86AS::GS, llvm::SDNode::hasOneUse(), llvm::Offset, Ptr, llvm::MachineFrameInfo::setHasCalls(), llvm::X86ISD::TLSADDR, llvm::X86ISD::TLSBASEADDR, llvm::X86ISD::TLSDESC, llvm::SDNode::user_begin(), and llvm::TargetMachine::useTLSDESC().

Referenced by LowerToTLSGeneralDynamicModel32(), LowerToTLSGeneralDynamicModel64(), LowerToTLSGeneralDynamicModelX32(), and LowerToTLSLocalDynamicModel().

getUnderlyingExtractedFromVec()

static int getUnderlyingExtractedFromVec ( SDValue &  ExtractedFromVec, SDValue  ExtIdx  )

static

For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.

Modifies ExtractedFromVec to the real vector and returns the real index.

Definition at line 7679 of file X86ISelLowering.cpp.

References assert(), llvm::SDNode::getAsZExtVal(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), Idx, and isUndefOrInRange().

Referenced by buildFromShuffleMostly().

getUnpackh()

static SDValue getUnpackh ( SelectionDAG &  DAG, const SDLoc &  dl, EVT  VT, SDValue  V1, SDValue  V2  )

static

Returns a vector_shuffle node for an unpackh operation.

Definition at line 4711 of file X86ISelLowering.cpp.

References llvm::createUnpackShuffleMask(), and getVectorShuffle().

Referenced by combineArithReduction(), convertShiftLeftToScale(), LowerAVXExtend(), LowerFunnelShift(), LowerHorizontalByteSum(), LowerMUL(), LowerRotate(), LowerShift(), and LowervXi8MulWithUNPCK().

getUnpackl()

static SDValue getUnpackl ( SelectionDAG &  DAG, const SDLoc &  dl, EVT  VT, SDValue  V1, SDValue  V2  )

static

Returns a vector_shuffle node for an unpackl operation.

Definition at line 4703 of file X86ISelLowering.cpp.

References llvm::createUnpackShuffleMask(), and getVectorShuffle().

Referenced by combineArithReduction(), convertShiftLeftToScale(), LowerFunnelShift(), LowerHorizontalByteSum(), LowerMUL(), LowerRotate(), LowerShift(), LowerUINT_TO_FP_i64(), and LowervXi8MulWithUNPCK().

getV4X86ShuffleImm()

static unsigned getV4X86ShuffleImm ( ArrayRef< int >  Mask )

static

Get a 4-lane 8-bit shuffle immediate for a mask.

This helper function produces an 8-bit shuffle immediate corresponding to the ubiquitous shuffle encoding scheme used in x86 instructions for shuffling 4 lanes. It can be used with most of the PSHUF instructions for example.

NB: We rely heavily on "undef" masks preserving the input lane.

Definition at line 9937 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), and llvm::find_if().

Referenced by combineVectorShiftImm(), getV4X86ShuffleImm8ForMask(), matchBinaryPermuteShuffle(), and matchUnaryPermuteShuffle().

getV4X86ShuffleImm8ForMask()

static SDValue getV4X86ShuffleImm8ForMask ( ArrayRef< int >  Mask, const SDLoc &  DL, SelectionDAG &  DAG  )

static

Definition at line 9961 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getTargetConstant(), and getV4X86ShuffleImm().

Referenced by canonicalizeShuffleMaskWithHorizOp(), combineConcatVectorOps(), combineRedundantDWordShuffle(), combineTargetShuffle(), combineVectorHADDSUB(), combineVectorShiftImm(), combineX86ShuffleChain(), LowerShift(), lowerShuffleAsSpecificZeroOrAnyExtend(), lowerShuffleWithSHUFPS(), lowerV16F32Shuffle(), lowerV16I32Shuffle(), lowerV2I64Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV4X128Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I16GeneralSingleInputShuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().

getVectorMaskingNode()

static SDValue getVectorMaskingNode ( SDValue  Op, SDValue  Mask, SDValue  PreservedSrc, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics.

Definition at line 25788 of file X86ISelLowering.cpp.

References getMaskNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::isAllOnesConstant(), llvm::SDValue::isUndef(), and llvm::ISD::VSELECT.

Referenced by lowerShuffleAsBlend().

getVectorShuffle()

static SDValue getVectorShuffle ( SelectionDAG &  DAG, EVT  VT, const SDLoc &  dl, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask  )

static

Definition at line 4682 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getBuildVector(), llvm::SDValue::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::ISD::isBuildVectorOfConstantSDNodes(), and llvm::SDValue::isUndef().

Referenced by getUnpackh(), and getUnpackl().

getVShift()

static SDValue getVShift ( bool  isLeft, EVT  VT, SDValue  SrcOp, unsigned  NumBits, SelectionDAG &  DAG, const TargetLowering &  TLI, const SDLoc &  dl  )

static

Return a vector logical shift node.

Definition at line 6925 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::EVT::is128BitVector(), llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.

getX86XALUOOp()

static std::pair< SDValue, SDValue > getX86XALUOOp ( X86::CondCode &  Cond, SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 24303 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADD, assert(), Cond, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_O, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::isOneConstant(), LHS, llvm_unreachable, RHS, llvm::ISD::SADDO, llvm::X86ISD::SMUL, llvm::ISD::SMULO, llvm::ISD::SSUBO, llvm::X86ISD::SUB, llvm::ISD::UADDO, llvm::X86ISD::UMUL, llvm::ISD::UMULO, and llvm::ISD::USUBO.

Referenced by LowerXALUO().

getZeroVector()

static SDValue getZeroVector ( MVT  VT, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Returns a vector of specified type with all zero elements.

Definition at line 3968 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), and llvm::TargetLoweringBase::isTypeLegal().

hasBZHI()

static bool hasBZHI ( const X86Subtarget &  Subtarget, MVT  VT  )

static

Definition at line 50483 of file X86ISelLowering.cpp.

Referenced by combineAndLoadToBZHI().

hasFPCMov()

static bool hasFPCMov ( unsigned  X86CC )

static

Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.

Definition at line 3025 of file X86ISelLowering.cpp.

References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, and llvm::X86::COND_P.

Referenced by combineCMov().

hasIdenticalHalvesShuffleMask()

static bool hasIdenticalHalvesShuffleMask ( ArrayRef< int >  Mask )

static

Return true if a shuffle mask chooses elements identically in its top and bottom halves.

For example, any splat mask has the same top and bottom halves. If an element is undefined in only one half of the mask, the halves are not considered identical.

Definition at line 9919 of file X86ISelLowering.cpp.

References assert().

Referenced by LowerAVXExtend().

hasNonFlagsUse()

static bool hasNonFlagsUse ( SDValue  Op )

static

return true if Op has a use that doesn't just read flags.

Definition at line 22880 of file X86ISelLowering.cpp.

References llvm::ISD::BRCOND, llvm::Use::getOperandNo(), llvm::Use::getUser(), llvm::Value::hasOneUse(), llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::TRUNCATE, and llvm::Value::use_begin().

incDecVectorConstant()

static SDValue incDecVectorConstant ( SDValue  V, SelectionDAG &  DAG, bool  IsInc, bool  NSW  )

static

Given a buildvector constant, return a new vector constant with each element incremented or decremented.

If incrementing or decrementing would result in unsigned overflow or underflow or this is not a simple vector constant, return an empty value.

Definition at line 23471 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::APInt::isMaxSignedValue(), llvm::APInt::isMaxValue(), llvm::APInt::isMinSignedValue(), llvm::APInt::isZero(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by combineSetCC(), LowerVSETCC(), and LowerVSETCCWithSUBUS().

insert128BitVector()

static SDValue insert128BitVector ( SDValue  Result, SDValue  Vec, unsigned  IdxVal, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Generate a DAG to put 128-bits into a vector > 128 bits.

This sets things up to match to an AVX VINSERTF128/VINSERTI128 or AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a simple superregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering INSERT_VECTOR_ELT operations easier.

Definition at line 4099 of file X86ISelLowering.cpp.

References assert(), llvm::SDValue::getValueType(), insertSubVector(), and llvm::EVT::is128BitVector().

Referenced by LowerSCALAR_TO_VECTOR(), and lowerV4X128Shuffle().

insert1BitVector()

static SDValue insert1BitVector ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Insert i1-subvector to i1-vector.

Definition at line 4429 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::flipAllBits(), llvm::APInt::getBitsSet(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), High, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), llvm::X86ISD::KSHIFTL, llvm::X86ISD::KSHIFTR, llvm::Low, llvm::SDNode::ops(), llvm::ISD::OR, and widenMaskVectorType().

Referenced by LowerINSERT_SUBVECTOR().

InsertBitToMaskVector()

static SDValue InsertBitToMaskVector ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Insert one bit to mask vector, like v16i1 or v8i1.

AVX-512 feature.

Definition at line 18481 of file X86ISelLowering.cpp.

References llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SIGN_EXTEND, and llvm::ISD::TRUNCATE.

insertSubVector()

static SDValue insertSubVector ( SDValue  Result, SDValue  Vec, unsigned  IdxVal, SelectionDAG &  DAG, const SDLoc &  dl, unsigned  vectorWidth  )

static

Definition at line 4069 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_SUBVECTOR, llvm::isPowerOf2_32(), and llvm::SDValue::isUndef().

Referenced by combineSelect(), combineX86ShuffleChain(), concatSubVectors(), getHopForBuildVector(), insert128BitVector(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

is128BitLaneCrossingShuffleMask()

static bool is128BitLaneCrossingShuffleMask ( MVT  VT, ArrayRef< int >  Mask  )

static

Test whether there are elements crossing 128-bit lanes in this shuffle mask.

Definition at line 9585 of file X86ISelLowering.cpp.

References llvm::MVT::getScalarSizeInBits(), and isLaneCrossingShuffleMask().

Referenced by combineX86ShuffleChain(), lowerShuffleAsBlendOfPSHUFBs(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), and matchUnaryPermuteShuffle().

is128BitLaneRepeatedShuffleMask() [1/2]

static bool is128BitLaneRepeatedShuffleMask ( MVT  VT, ArrayRef< int >  Mask  )

static

Definition at line 9665 of file X86ISelLowering.cpp.

References isRepeatedShuffleMask().

is128BitLaneRepeatedShuffleMask() [2/2]

static bool is128BitLaneRepeatedShuffleMask ( MVT  VT, ArrayRef< int >  Mask, SmallVectorImpl< int > &  RepeatedMask  )

static

Test whether a shuffle mask is equivalent within each 128-bit lane.

Definition at line 9659 of file X86ISelLowering.cpp.

References isRepeatedShuffleMask().

Referenced by combineBlendOfPermutes(), LowerShift(), lowerShuffleAsBlend(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV32I16Shuffle(), lowerV4I64Shuffle(), lowerV8F32Shuffle(), lowerV8I32Shuffle(), lowerV8I64Shuffle(), matchShuffleAsByteRotate(), and matchUnaryPermuteShuffle().

is128BitUnpackShuffleMask()

static bool is128BitUnpackShuffleMask ( ArrayRef< int >  Mask, const SelectionDAG &  DAG  )

static

Definition at line 9894 of file X86ISelLowering.cpp.

References llvm::ShuffleVectorSDNode::commuteMask(), llvm::createUnpackShuffleMask(), llvm::MVT::getIntegerVT(), llvm::MVT::getVectorVT(), and isTargetShuffleEquivalent().

Referenced by lowerShuffleOfExtractsAsVperm(), and lowerShuffleWithUndefHalf().

is256BitLaneRepeatedShuffleMask()

static bool is256BitLaneRepeatedShuffleMask ( MVT  VT, ArrayRef< int >  Mask, SmallVectorImpl< int > &  RepeatedMask  )

static

Test whether a shuffle mask is equivalent within each 256-bit lane.

Definition at line 9672 of file X86ISelLowering.cpp.

References isRepeatedShuffleMask().

Referenced by lowerV8F64Shuffle(), lowerV8I64Shuffle(), and matchUnaryPermuteShuffle().

isAddSubOrSubAdd() [1/2]

static bool isAddSubOrSubAdd ( const BuildVectorSDNode *  BV, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, SDValue &  Opnd0, SDValue &  Opnd1, unsigned &  NumExtracts, bool &  IsSubAdd  )

static

Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB/SUBADD operation.

If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1 (SUBADD = Opnd0 -+ Opnd1) operation are written to the parameters Opnd0 and Opnd1.

Definition at line 8074 of file X86ISelLowering.cpp.

References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE3(), llvm::MVT::isFloatingPoint(), llvm::SDValue::isUndef(), std::swap(), and llvm::ISD::UNDEF.

Referenced by combineShuffleToAddSubOrFMAddSub(), and lowerToAddSubOrFMAddSub().

isAddSubOrSubAdd() [2/2]

static bool isAddSubOrSubAdd ( SDNode *  N, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, SDValue &  Opnd0, SDValue &  Opnd1, bool &  IsSubAdd  )

static

Returns true iff the shuffle node N can be replaced with ADDSUB(SUBADD) operation.

If true is returned then the operands of ADDSUB(SUBADD) operation are written to the parameters Opnd0 and Opnd1.

We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes so it is easier to generically match. We also insert dummy vector shuffle nodes for the operands which explicitly discard the lanes which are unused by this operation to try to flow through the rest of the combiner the fact that they're unused.

Definition at line 42477 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE3(), isAddSubOrSubAddMask(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, RHS, and llvm::ISD::VECTOR_SHUFFLE.

isAddSubOrSubAddMask()

static bool isAddSubOrSubAddMask ( ArrayRef< int >  Mask, bool &  Op0Even  )

static

Checks if the shuffle mask takes subsequent elements alternately from two vectors.

For example <0, 5, 2, 7> or <8, 1, 10, 3, 12, 5, 14, 7> are both correct.

Definition at line 42440 of file X86ISelLowering.cpp.

References Size.

Referenced by combineShuffleToFMAddSub(), and isAddSubOrSubAdd().

isAndOrOfSetCCs()

static bool isAndOrOfSetCCs ( SDValue  Op, unsigned &  Opc  )

static

Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR.

Definition at line 25222 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::OR, and llvm::X86ISD::SETCC.

isAnyInRange()

static bool isAnyInRange ( ArrayRef< int >  Mask, int  Low, int  Hi  )

static

Return true if the value of any element in Mask falls within the specified range (L, H].

Definition at line 3681 of file X86ISelLowering.cpp.

References llvm::any_of(), llvm::Hi, isInRange(), and llvm::Low.

Referenced by getTargetConstantBitsFromNode().

isAnyZero()

static bool isAnyZero ( ArrayRef< int >  Mask )

static

Return true if the value of any element in Mask is the zero sentinel value.

Definition at line 3686 of file X86ISelLowering.cpp.

References llvm::any_of(), and llvm::SM_SentinelZero.

Referenced by combineHorizOpWithShuffle(), combineX86ShuffleChain(), getTargetShuffleMask(), isHorizontalBinOp(), matchBinaryPermuteShuffle(), matchShuffleAsByteRotate(), and matchUnaryPermuteShuffle().

isBlendOrUndef()

static LLVM_ATTRIBUTE_UNUSED bool isBlendOrUndef ( ArrayRef< int >  Mask )

static

Return true if every element in Mask, is an in-place blend/select mask or is undef.

Definition at line 3718 of file X86ISelLowering.cpp.

References llvm::enumerate(), I, and isUndefOrEqual().

Referenced by combineBlendOfPermutes().

isBroadcastShuffleMask()

static bool isBroadcastShuffleMask ( ArrayRef< int >  Mask )

static

Definition at line 11323 of file X86ISelLowering.cpp.

References isUndefOrEqual().

Referenced by isNoopOrBroadcastShuffleMask(), lower1BitShuffle(), and lowerShuffleAsDecomposedShuffleMerge().

isCMOVPseudo()

static bool isCMOVPseudo ( MachineInstr &  MI )

static

Definition at line 35578 of file X86ISelLowering.cpp.

References MI.

isCompletePermute()

static bool isCompletePermute ( ArrayRef< int >  Mask )

static

Return true if every element of a single input is referenced by the shuffle mask.

i.e. it just permutes them all.

Definition at line 3758 of file X86ISelLowering.cpp.

References llvm::APInt::getZero(), llvm::APInt::isAllOnes(), isInRange(), and llvm::APInt::setBit().

Referenced by combineSetCCMOVMSK().

isConstantPowerOf2()

static bool isConstantPowerOf2 ( SDValue  V, unsigned  EltSizeInBIts, bool  AllowUndefs  )

static

Definition at line 5212 of file X86ISelLowering.cpp.

References getTargetConstantBitsFromNode(), I, and llvm::SmallVectorBase< Size_T >::size().

Referenced by combineOrXorWithSETCC(), combineSelect(), and LowerVSETCC().

isEFLAGSLiveAfter()

static bool isEFLAGSLiveAfter ( MachineBasicBlock::iterator  Itr, MachineBasicBlock *  BB  )

static

Definition at line 35175 of file X86ISelLowering.cpp.

References llvm::MachineBasicBlock::end(), llvm::make_range(), and llvm::MachineBasicBlock::successors().

Referenced by checkAndUpdateEFLAGSKill(), and emitXBegin().

IsElementEquivalent()

static bool IsElementEquivalent ( int  MaskSize, SDValue  Op, SDValue  ExpectedOp, int  Idx, int  ExpectedIdx  )

static

Checks whether the vector elements referenced by two shuffle masks are equivalent.

Definition at line 9725 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::SDValue::getNumOperands(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::X86ISD::HSUB, Idx, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VBROADCAST_LOAD.

Referenced by isShuffleEquivalent(), isTargetShuffleEquivalent(), and matchShuffleAsBlend().

isFMAddSubOrFMSubAdd()

static bool isFMAddSubOrFMSubAdd ( const X86Subtarget &  Subtarget, SelectionDAG &  DAG, SDValue &  Opnd0, SDValue &  Opnd1, SDValue &  Opnd2, unsigned  ExpectedUses  )

static

Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBADD(Opnd0, Opnd1) into FMADDSUB/FMSUBADD(x, y, Opnd1).

If (and only if) true is returned, the operands of FMADDSUB/FMSUBADD are written to parameters Opnd0, Opnd1, Opnd2.

Prior to calling this function it should be known that there is some SDNode that potentially can be replaced with an X86ISD::ADDSUB operation using Opnd0 and Opnd1 as operands. Also, this method is called before replacement of such SDNode with ADDSUB operation. Thus the number of Opnd0 uses is expected to be equal to 2. For example, this function may be called for the following IR: AB = fmul fast <2 x double> A, B Sub = fsub fast <2 x double> AB, C Add = fadd fast <2 x double> AB, C Addsub = shufflevector <2 x double> Sub, <2 x double> Add, <2 x i32> <i32 0, i32 3> There is a def for Addsub here, which potentially can be replaced by X86ISD::ADDSUB operation: Addsub = X86ISD::ADDSUB AB, C and such ADDSUB can further be replaced with FMADDSUB: Addsub = FMADDSUB A, B, C.

The main reason why this method is called before the replacement of the recognized ADDSUB idiom with ADDSUB operation is that such replacement is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit FMADDSUB is.

Definition at line 8201 of file X86ISelLowering.cpp.

References llvm::FPOpFusion::Fast, llvm::ISD::FMUL, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTarget(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDNode::hasNUsesOfValue(), llvm::TargetMachine::Options, and Options.

Referenced by combineShuffleToAddSubOrFMAddSub(), and lowerToAddSubOrFMAddSub().

isFNEG()

static SDValue isFNEG ( SelectionDAG &  DAG, SDNode *  N, unsigned  Depth = 0  )

static

Returns the negated value if the node N flips sign of FP value.

FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000) or FSUB(0, x) AVX512F does not have FXOR, so FNEG is lowered as (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))). In this case we go though all bitcasts. This also recognizes splat of a negated value and returns the splat of that value.

Definition at line 53757 of file X86ISelLowering.cpp.

References llvm::Depth, llvm::ISD::FNEG, llvm::ISD::FSUB, llvm::X86ISD::FXOR, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::ISD::INSERT_VECTOR_ELT, isFNEG(), llvm::SDValue::isUndef(), llvm::SelectionDAG::MaxRecursionDepth, N, llvm::peekThroughBitcasts(), llvm::SmallVectorBase< Size_T >::size(), std::swap(), llvm::ISD::VECTOR_SHUFFLE, and llvm::ISD::XOR.

Referenced by combineFneg(), llvm::X86TargetLowering::getNegatedExpression(), and isFNEG().

isFoldableUseOfShuffle()

static bool isFoldableUseOfShuffle ( SDNode *  N )

static

Definition at line 7411 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, isFoldableUseOfShuffle(), isTargetShuffle(), N, llvm::X86ISD::VPDPBUSD, llvm::X86ISD::VPERMV, and llvm::X86ISD::VPERMV3.

Referenced by isFoldableUseOfShuffle(), and lowerBuildVectorAsBroadcast().

isFRClass()

static bool isFRClass ( const TargetRegisterClass &  RC )

static

Check if RC is a vector register class.

I.e., FR* / VR* or one of their variant.

Definition at line 60097 of file X86ISelLowering.cpp.

References llvm::TargetRegisterClass::hasSuperClassEq().

Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().

isFreeToSplitVector()

static bool isFreeToSplitVector ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 4240 of file X86ISelLowering.cpp.

References collectConcatOps(), and N.

Referenced by LowerStore(), matchTruncateWithPACK(), narrowExtractedVectorSelect(), and narrowVectorSelect().

isGRClass()

static bool isGRClass ( const TargetRegisterClass &  RC )

static

Check if RC is a general purpose register class.

I.e., GR* or one of their variant.

Definition at line 60087 of file X86ISelLowering.cpp.

References llvm::TargetRegisterClass::hasSuperClassEq().

Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().

isHopBuildVector()

static bool isHopBuildVector ( const BuildVectorSDNode *  BV, SelectionDAG &  DAG, unsigned &  HOpcode, SDValue &  V0, SDValue &  V1  )

static

Definition at line 8269 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::DELETED_NODE, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::X86ISD::HSUB, llvm::MVT::is256BitVector(), llvm::SDValue::isUndef(), and llvm::ISD::SUB.

Referenced by LowerToHorizontalOp().

isHorizontalBinOp()

static bool isHorizontalBinOp ( unsigned  HOpcode, SDValue &  LHS, SDValue &  RHS, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, bool  IsCommutative, SmallVectorImpl< int > &  PostShuffleMask, bool  ForceHorizOp  )

static

Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS.

A horizontal operation performs the binary operation on successive elements of its first operand, then on successive elements of its second operand, returning the resulting values in a vector. For example, if A = < float a0, float a1, float a2, float a3 > and B = < float b0, float b1, float b2, float b3 > then the result of doing a horizontal operation on A and B is A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >. In short, LHS and RHS are inspected to see if LHS op RHS is of the form A horizontal-op B, for some already available A and B, and if so then LHS is set to A, RHS to B, and the routine returns 'true'.

Definition at line 52981 of file X86ISelLowering.cpp.

References A, llvm::all_of(), llvm::any_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::assign(), B, llvm::sampleprof::Base, llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::CallingConv::C, llvm::SmallVectorImpl< T >::clear(), llvm::ShuffleVectorSDNode::commuteMask(), D, llvm::SmallVectorBase< Size_T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), getTargetShuffleInputs(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), isAnyZero(), llvm::MVT::isFloatingPoint(), isMultiLaneShuffleMask(), llvm::isNullConstant(), isSequentialOrUndefInRange(), isUndefOrInRange(), LHS, llvm::peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveTargetShuffleInputsAndMask(), RHS, scaleShuffleElements(), shouldUseHorizontalOp(), llvm::SmallVectorBase< Size_T >::size(), llvm::ArrayRef< T >::slice(), llvm::SM_SentinelUndef, llvm::SelectionDAG::SplitVector(), std::swap(), and llvm::SDNode::users().

Referenced by combineToHorizontalAddSub().

isHorizontalBinOpPart()

static bool isHorizontalBinOpPart ( const BuildVectorSDNode *  N, unsigned  Opcode, const SDLoc &  DL, SelectionDAG &  DAG, unsigned  BaseIdx, unsigned  LastIdx, SDValue &  V0, SDValue &  V1  )

static

This is a helper function of LowerToHorizontalOp().

This function checks that the build_vector N in input implements a 128-bit partial horizontal operation on a 256-bit vector, but that operation may not match the layout of an x86 256-bit horizontal instruction. In other words, if this returns true, then some extraction/insertion will be required to produce a valid horizontal instruction.

Parameter Opcode defines the kind of horizontal operation to match. For example, if Opcode is equal to ISD::ADD, then this function checks if N implements a horizontal arithmetic add; if instead Opcode is equal to ISD::SUB, then this function checks if this is a horizontal arithmetic sub.

This function only analyzes elements of N whose indices are in range [BaseIdx, LastIdx).

TODO: This function was originally used to match both real and fake partial horizontal operations, but the index-matching logic is incorrect for that. See the corrected implementation in isHopBuildVector(). Can we reduce this code because it is only used for partial h-op matching now?

Definition at line 7919 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::is256BitVector(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), and N.

Referenced by LowerToHorizontalOp().

isHorizOp()

static LLVM_ATTRIBUTE_UNUSED bool isHorizOp ( unsigned  Opcode )

static

Definition at line 7886 of file X86ISelLowering.cpp.

References llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::X86ISD::HADD, llvm::X86ISD::HSUB, llvm::X86ISD::PACKSS, and llvm::X86ISD::PACKUS.

Referenced by combineHorizOpWithShuffle().

isInRange()

static bool isInRange ( int  Val, int  Low, int  Hi  )

static

Return true if Val falls within the specified range (L, H].

Definition at line 3675 of file X86ISelLowering.cpp.

References llvm::Hi, and llvm::Low.

Referenced by canonicalizeShuffleMaskWithHorizOp(), combineX86ShuffleChain(), combineX86ShufflesRecursively(), getFauxShuffleMask(), isAnyInRange(), isCompletePermute(), isTargetShuffleEquivalent(), isUndefOrInRange(), and isUndefOrZeroOrInRange().

isLaneCrossingShuffleMask()

static bool isLaneCrossingShuffleMask ( unsigned  LaneSizeInBits, unsigned  ScalarSizeInBits, ArrayRef< int >  Mask  )

static

Test whether there are elements crossing LaneSizeInBits lanes in this shuffle mask.

X86 divides up its shuffles into in-lane and cross-lane shuffle operations and we routinely test for these.

Definition at line 9569 of file X86ISelLowering.cpp.

References assert(), and Size.

Referenced by combineBlendOfPermutes(), and is128BitLaneCrossingShuffleMask().

isLegalConversion()

static bool isLegalConversion ( MVT  VT, bool  IsSigned, const X86Subtarget &  Subtarget  )

static

Definition at line 19633 of file X86ISelLowering.cpp.

References llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE2(), and llvm::X86Subtarget::useAVX512Regs().

isLogicOp()

static bool isLogicOp ( unsigned  Opcode )

static

Definition at line 2789 of file X86ISelLowering.cpp.

References llvm::X86ISD::ANDNP, and llvm::ISD::isBitwiseLogicOp().

Referenced by canonicalizeShuffleWithOp().

isMultiLaneShuffleMask()

static bool isMultiLaneShuffleMask ( unsigned  LaneSizeInBits, unsigned  ScalarSizeInBits, ArrayRef< int >  Mask  )

static

Test whether elements in each LaneSizeInBits lane in this shuffle mask come from multiple lanes - this is different to isLaneCrossingShuffleMask to better support 'repeated mask + lane permute' style shuffles.

Definition at line 9592 of file X86ISelLowering.cpp.

References assert().

Referenced by isHorizontalBinOp().

isNonZeroElementsInOrder()

static bool isNonZeroElementsInOrder ( const APInt &  Zeroable, ArrayRef< int >  Mask, const EVT &  VectorType, bool &  IsZeroSideLeft  )

static

Definition at line 10010 of file X86ISelLowering.cpp.

References assert().

Referenced by lowerShuffleWithEXPAND().

isNoopOrBroadcastShuffleMask()

static bool isNoopOrBroadcastShuffleMask ( ArrayRef< int >  Mask )

static

Definition at line 11327 of file X86ISelLowering.cpp.

References isBroadcastShuffleMask(), and isNoopShuffleMask().

Referenced by lowerShuffleAsDecomposedShuffleMerge().

isNoopShuffleMask()

static bool isNoopShuffleMask ( ArrayRef< int >  Mask )

static

Tiny helper function to identify a no-op mask.

This is a somewhat boring predicate function. It checks whether the mask array input, which is assumed to be a single-input shuffle mask of the kind used by the X86 shuffle instructions (not a fully general ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an in-place shuffle are 'no-op's.

Definition at line 9555 of file X86ISelLowering.cpp.

References assert(), and Size.

Referenced by combineTargetShuffle(), isNoopOrBroadcastShuffleMask(), lowerShuffleAsByteRotate(), lowerShuffleAsByteShiftMask(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsPermuteAndUnpack(), lowerV8I16GeneralSingleInputShuffle(), and matchShuffleAsBitRotate().

IsNOT()

static SDValue IsNOT ( SDValue  V, SelectionDAG &  DAG  )

static

Definition at line 5228 of file X86ISelLowering.cpp.

References llvm::ISD::AND, collectConcatOps(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::isAllOnesConstant(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), IsNOT(), llvm::isNullConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::OR, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), and llvm::ISD::XOR.

Referenced by combineAndNotIntoANDNP(), combineAndnp(), combineAndShuffleNot(), combineMOVMSK(), combinePTESTCC(), combineSelect(), combineVectorPack(), and IsNOT().

isNullFPScalarOrVectorConst()

static bool isNullFPScalarOrVectorConst ( SDValue  V )

static

Definition at line 54268 of file X86ISelLowering.cpp.

References llvm::ISD::isBuildVectorAllZeros(), and llvm::isNullFPConstant().

Referenced by combineFAndn(), combineFOr(), combineSetCC(), and getNullFPConstForNullVal().

isOrXorXorTree()

static bool isOrXorXorTree ( SDValue  X, bool  Root = true  )

static

Recursive helper for combineVectorSizedSetCCEquality() to see if we have a recognizable memcmp expansion.

Definition at line 22346 of file X86ISelLowering.cpp.

References isOrXorXorTree(), llvm::ISD::OR, X, and llvm::ISD::XOR.

Referenced by combineVectorSizedSetCCEquality(), and isOrXorXorTree().

isProfitableToUseFlagOp()

static bool isProfitableToUseFlagOp ( SDValue  Op )

static

Definition at line 22900 of file X86ISelLowering.cpp.

References llvm::ISD::CopyToReg, llvm::ISD::SETCC, and llvm::ISD::STORE.

Referenced by EmitTest().

isRepeatedShuffleMask()

static bool isRepeatedShuffleMask ( unsigned  LaneSizeInBits, MVT  VT, ArrayRef< int >  Mask, SmallVectorImpl< int > &  RepeatedMask  )

static

Test whether a shuffle mask is equivalent within each sub-lane.

This checks a shuffle mask to see if it is performing the same lane-relative shuffle in each sub-lane. This trivially implies that it is also not lane-crossing. It may however involve a blend from the same lane of a second vector.

The specific repeated shuffle mask is populated in RepeatedMask, as it is non-trivial to compute in the face of undef lanes. The representation is suitable for use with existing 128-bit shuffles as entries from the second vector have been remapped to [LaneSize, 2*LaneSize).

Definition at line 9629 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::MVT::getScalarSizeInBits(), Size, and llvm::SM_SentinelUndef.

Referenced by is128BitLaneRepeatedShuffleMask(), and is256BitLaneRepeatedShuffleMask().

isRepeatedTargetShuffleMask() [1/2]

static bool isRepeatedTargetShuffleMask ( unsigned  LaneSizeInBits, MVT  VT, ArrayRef< int >  Mask, SmallVectorImpl< int > &  RepeatedMask  )

static

Test whether a target shuffle mask is equivalent within each sub-lane.

Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.

Definition at line 9716 of file X86ISelLowering.cpp.

References llvm::MVT::getScalarSizeInBits(), and isRepeatedTargetShuffleMask().

isRepeatedTargetShuffleMask() [2/2]

static bool isRepeatedTargetShuffleMask ( unsigned  LaneSizeInBits, unsigned  EltSizeInBits, ArrayRef< int >  Mask, SmallVectorImpl< int > &  RepeatedMask  )

static

Test whether a target shuffle mask is equivalent within each sub-lane.

Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.

Definition at line 9679 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), isUndefOrZero(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by canonicalizeShuffleMaskWithHorizOp(), isRepeatedTargetShuffleMask(), and matchBinaryPermuteShuffle().

isSequentialOrUndefInRange()

static bool isSequentialOrUndefInRange ( ArrayRef< int >  Mask, unsigned  Pos, unsigned  Size, int  Low, int  Step = 1  )

static

Return true if every element in Mask, beginning from position Pos and ending in Pos + Size, falls within the specified sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.

Definition at line 3729 of file X86ISelLowering.cpp.

References isUndefOrEqual(), llvm::Low, and Size.

Referenced by combineX86ShuffleChain(), isHorizontalBinOp(), lowerShuffleAsByteShiftMask(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsVALIGN(), lowerShuffleAsVTRUNC(), lowerShuffleAsVTRUNCAndUnpack(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithUndefHalf(), lowerShuffleWithVPMOV(), lowerV8I16GeneralSingleInputShuffle(), match1BitShuffleAsKSHIFT(), matchShuffleAsINSERTQ(), matchShuffleAsShift(), matchShuffleAsVTRUNC(), matchUnaryPermuteShuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

isSequentialOrUndefOrZeroInRange()

static bool isSequentialOrUndefOrZeroInRange ( ArrayRef< int >  Mask, unsigned  Pos, unsigned  Size, int  Low, int  Step = 1  )

static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero.

Definition at line 3740 of file X86ISelLowering.cpp.

References isUndefOrZero(), llvm::Low, and Size.

Referenced by combineX86ShuffleChain(), and matchShuffleWithUNPCK().

isShuffleEquivalent()

static bool isShuffleEquivalent ( ArrayRef< int >  Mask, ArrayRef< int >  ExpectedMask, SDValue  V1 = SDValue(), SDValue  V2 = SDValue()  )

static

Checks whether a shuffle mask is equivalent to an explicit list of arguments.

This is a fast way to test a shuffle mask against a fixed pattern:

if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }

It returns true if the mask is exactly as wide as the argument list, and each element of the mask is either -1 (signifying undef) or the value given in the argument.

Definition at line 9785 of file X86ISelLowering.cpp.

References assert(), IsElementEquivalent(), llvm::ArrayRef< T >::size(), and Size.

Referenced by lowerShuffleWithUNPCK(), lowerShuffleWithUNPCK256(), lowerV16F32Shuffle(), lowerV2F64Shuffle(), lowerV2X128Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4X128Shuffle(), lowerV8F32Shuffle(), and lowerV8F64Shuffle().

isShuffleFoldableLoad()

static bool isShuffleFoldableLoad ( SDValue  V )

static

Helper to test for a load that can be folded with x86 shuffles.

This is particularly important because the set of instructions varies significantly based on whether the operand is a load or not.

Definition at line 12404 of file X86ISelLowering.cpp.

References getNode(), llvm::ISD::isNON_EXTLoad(), and llvm::peekThroughOneUseBitcasts().

Referenced by canonicalizeShuffleWithOp(), combineTargetShuffle(), lowerShuffleAsBroadcast(), and lowerShuffleWithPERMV().

isShuffleMaskInputInPlace()

static bool isShuffleMaskInputInPlace ( int  Input, ArrayRef< int >  Mask  )

static

Test whether the specified input (0 or 1) is in-place blended by the given mask.

This returns true if the elements from a particular input are already in the slot required by the given mask and require no permutation.

Definition at line 12622 of file X86ISelLowering.cpp.

References assert(), and Size.

Referenced by lowerV4F64Shuffle(), and lowerV4I64Shuffle().

isSingleElementRepeatedMask()

static bool isSingleElementRepeatedMask ( ArrayRef< int >  Mask )

static

Check if the Mask consists of the same element repeated multiple times.

Definition at line 11332 of file X86ISelLowering.cpp.

References llvm::SM_SentinelUndef.

Referenced by lowerShuffleAsDecomposedShuffleMerge().

isSingleSHUFPSMask()

static bool isSingleSHUFPSMask ( ArrayRef< int >  Mask )

static

Test whether this can be lowered with a single SHUFPS instruction.

This is used to disable more specialized lowerings when the shufps lowering will happen to be efficient.

Definition at line 12599 of file X86ISelLowering.cpp.

References assert().

Referenced by lowerShuffleOfExtractsAsVperm(), lowerShuffleWithUndefHalf(), lowerV16I32Shuffle(), lowerV4F32Shuffle(), lowerV4I32Shuffle(), and lowerV8I32Shuffle().

isSoftF16()

template<typename T >

static bool isSoftF16 ( T  VT, const X86Subtarget &  Subtarget  )

static

Definition at line 12410 of file X86ISelLowering.cpp.

Referenced by combineFMinNumFMaxNum(), combineSelect(), combineShuffle(), lowerShuffleAsElementInsertion(), LowerVSETCC(), and llvm::X86TargetLowering::ReplaceNodeResults().

isTargetShuffle()

static bool isTargetShuffle ( unsigned  Opcode )

static

Definition at line 2794 of file X86ISelLowering.cpp.

References llvm::X86ISD::BLENDI, llvm::X86ISD::EXTRQI, llvm::X86ISD::INSERTPS, llvm::X86ISD::INSERTQI, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VALIGN, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VZEXT_MOVL.

Referenced by canonicalizeShuffleWithOp(), combineINSERT_SUBVECTOR(), combineSelect(), combineShuffle(), combineTargetShuffle(), llvm::X86TargetLowering::computeKnownBitsForTargetNode(), llvm::X86TargetLowering::ComputeNumSignBitsForTargetNode(), getShuffleScalarElt(), getTargetShuffleAndZeroables(), getTargetShuffleMask(), and isFoldableUseOfShuffle().

isTargetShuffleEquivalent()

static bool isTargetShuffleEquivalent ( MVT  VT, ArrayRef< int >  Mask, ArrayRef< int >  ExpectedMask, const SelectionDAG &  DAG, SDValue  V1 = SDValue(), SDValue  V2 = SDValue()  )

static

Checks whether a target shuffle mask is equivalent to an explicit pattern.

The masks must be exactly the same width.

If an element in Mask matches SM_SentinelUndef (-1) then the corresponding value in ExpectedMask is always accepted. Otherwise the indices must match.

SM_SentinelZero is accepted as a valid negative index but must match in both, or via a known bits test.

Definition at line 9817 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), IsElementEquivalent(), isInRange(), isUndefOrZeroOrInRange(), llvm::APInt::isZero(), llvm::SelectionDAG::MaskedVectorIsZero(), llvm::APInt::setBit(), llvm::ArrayRef< T >::size(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineX86ShuffleChain(), is128BitUnpackShuffleMask(), isUnpackWdShuffleMask(), matchBinaryShuffle(), matchShuffleWithPACK(), matchShuffleWithUNPCK(), and matchUnaryShuffle().

isTargetShuffleVariableMask()

static bool isTargetShuffleVariableMask ( unsigned  Opcode )

static

Definition at line 2835 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::ISD::OR, llvm::X86ISD::PSHUFB, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, and llvm::X86ISD::VPPERM.

Referenced by combineX86ShufflesRecursively().

isTruncWithZeroHighBitsInput()

static bool isTruncWithZeroHighBitsInput ( SDValue  V, SelectionDAG &  DAG  )

static

Definition at line 24378 of file X86ISelLowering.cpp.

References llvm::APInt::getHighBitsSet(), llvm::SDValue::getValueSizeInBits(), llvm::SelectionDAG::MaskedValueIsZero(), and llvm::ISD::TRUNCATE.

isUnaryOp()

static bool isUnaryOp ( unsigned  Opcode )

static

Definition at line 41365 of file X86ISelLowering.cpp.

References llvm::ISD::CTLZ, llvm::ISD::CTPOP, and llvm::ISD::CTTZ.

Referenced by canonicalizeShuffleWithOp().

isUndefInRange()

static bool isUndefInRange ( ArrayRef< int >  Mask, unsigned  Pos, unsigned  Size  )

static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value.

Definition at line 3657 of file X86ISelLowering.cpp.

References llvm::all_of(), and Size.

Referenced by canonicalizeShuffleMaskWithHorizOp(), isUndefLowerHalf(), isUndefUpperHalf(), lowerShuffleAsVTRUNC(), lowerShuffleWithVPMOV(), matchBinaryPermuteShuffle(), matchShuffleAsINSERTQ(), matchUnaryShuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

isUndefLowerHalf()

static bool isUndefLowerHalf ( ArrayRef< int >  Mask )

static

Return true if the mask creates a vector whose lower half is undefined.

Definition at line 3663 of file X86ISelLowering.cpp.

References isUndefInRange().

Referenced by getHalfShuffleMask(), and lowerShuffleWithUndefHalf().

isUndefOrEqual() [1/2]

static bool isUndefOrEqual ( ArrayRef< int >  Mask, int  CmpVal  )

static

Return true if every element in Mask is the undef sentinel value or equal to the specified value.

Definition at line 3636 of file X86ISelLowering.cpp.

References llvm::all_of(), and llvm::SM_SentinelUndef.

isUndefOrEqual() [2/2]

static bool isUndefOrEqual ( int  Val, int  CmpVal  )

static

Val is the undef sentinel value or equal to the specified value.

Definition at line 3630 of file X86ISelLowering.cpp.

References llvm::SM_SentinelUndef.

Referenced by combineBlendOfPermutes(), combineX86ShuffleChain(), isBlendOrUndef(), isBroadcastShuffleMask(), isSequentialOrUndefInRange(), lowerShuffleAsLanePermuteAndPermute(), lowerV8I16GeneralSingleInputShuffle(), matchUnaryShuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

isUndefOrEqualInRange()

static bool isUndefOrEqualInRange ( ArrayRef< int >  Mask, int  CmpVal, unsigned  Pos, unsigned  Size  )

static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value or equal to the specified value.

Definition at line 3644 of file X86ISelLowering.cpp.

References llvm::all_of(), and Size.

Referenced by matchUnaryShuffle().

isUndefOrInRange() [1/2]

static bool isUndefOrInRange ( ArrayRef< int >  Mask, int  Low, int  Hi  )

static

Return true if every element in Mask is undef or if its value falls within the specified range (L, H].

Definition at line 3698 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::Hi, isUndefOrInRange(), and llvm::Low.

isUndefOrInRange() [2/2]

static bool isUndefOrInRange ( int  Val, int  Low, int  Hi  )

static

Return true if Val is undef or if its value falls within the specified range (L, H].

Definition at line 3692 of file X86ISelLowering.cpp.

References llvm::Hi, isInRange(), llvm::Low, and llvm::SM_SentinelUndef.

Referenced by combineBlendOfPermutes(), combineTargetShuffle(), combineX86ShuffleChain(), getFauxShuffleMask(), getUnderlyingExtractedFromVec(), isHorizontalBinOp(), isUndefOrInRange(), lowerShuffleAsByteShiftMask(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsUNPCKAndPermute(), lowerV8I16GeneralSingleInputShuffle(), matchBinaryPermuteShuffle(), and matchUnaryPermuteShuffle().

isUndefOrZero()

static bool isUndefOrZero ( int  Val )

static

Val is either the undef or zero sentinel value.

Definition at line 3651 of file X86ISelLowering.cpp.

References llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by canonicalizeShuffleMaskWithHorizOp(), combineSelect(), combineX86ShuffleChain(), combineX86ShufflesRecursively(), getTargetShuffleAndZeroables(), isRepeatedTargetShuffleMask(), isSequentialOrUndefOrZeroInRange(), isUndefOrZeroInRange(), isUndefOrZeroOrInRange(), matchShuffleWithUNPCK(), and matchUnaryShuffle().

isUndefOrZeroInRange()

static bool isUndefOrZeroInRange ( ArrayRef< int >  Mask, unsigned  Pos, unsigned  Size  )

static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero.

Definition at line 3751 of file X86ISelLowering.cpp.

References llvm::all_of(), isUndefOrZero(), and Size.

Referenced by combineExtractWithShuffle(), combineX86ShuffleChain(), matchBinaryPermuteShuffle(), matchUnaryShuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().

isUndefOrZeroOrInRange() [1/2]

static bool isUndefOrZeroOrInRange ( ArrayRef< int >  Mask, int  Low, int  Hi  )

static

Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H].

Definition at line 3711 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::Hi, isUndefOrZeroOrInRange(), and llvm::Low.

isUndefOrZeroOrInRange() [2/2]

static bool isUndefOrZeroOrInRange ( int  Val, int  Low, int  Hi  )

static

Return true if Val is undef, zero or if its value falls within the specified range (L, H].

Definition at line 3705 of file X86ISelLowering.cpp.

References llvm::Hi, isInRange(), isUndefOrZero(), and llvm::Low.

Referenced by canonicalizeShuffleMaskWithHorizOp(), isTargetShuffleEquivalent(), isUndefOrZeroOrInRange(), and matchShuffleWithSHUFPD().

isUndefUpperHalf()

static bool isUndefUpperHalf ( ArrayRef< int >  Mask )

static

Return true if the mask creates a vector whose upper half is undefined.

Definition at line 3669 of file X86ISelLowering.cpp.

References isUndefInRange().

Referenced by getHalfShuffleMask(), lowerShuffleAsSpecificZeroOrAnyExtend(), lowerShuffleWithUndefHalf(), matchShuffleAsEXTRQ(), matchShuffleAsINSERTQ(), and narrowShuffle().

isUnpackWdShuffleMask()

static bool isUnpackWdShuffleMask ( ArrayRef< int >  Mask, MVT  VT, const SelectionDAG &  DAG  )

static

Definition at line 9878 of file X86ISelLowering.cpp.

References llvm::createUnpackShuffleMask(), and isTargetShuffleEquivalent().

Referenced by lowerV8F32Shuffle(), and lowerV8I32Shuffle().

isUpperSubvectorUndef()

static SDValue isUpperSubvectorUndef ( SDValue  V, const SDLoc &  DL, SelectionDAG &  DAG  )

static

Definition at line 4219 of file X86ISelLowering.cpp.

References llvm::any_of(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), collectConcatOps(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by LowerTruncateVecPack(), and LowerTruncateVecPackWithSignBits().

isVKClass()

static bool isVKClass ( const TargetRegisterClass &  RC )

static

Check if RC is a mask register class.

I.e., VK* or one of their variant.

Definition at line 60108 of file X86ISelLowering.cpp.

References llvm::TargetRegisterClass::hasSuperClassEq().

Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().

isX86CCSigned()

static bool isX86CCSigned ( unsigned  X86CC )

static

Return true if the condition is an signed comparison operation.

Definition at line 2902 of file X86ISelLowering.cpp.

References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, and llvm_unreachable.

Referenced by EmitCmp().

isX86LogicalCmp()

static bool isX86LogicalCmp ( SDValue  Op )

static

Return true if opcode is a X86 logical comparison.

Definition at line 24364 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADC, llvm::X86ISD::ADD, llvm::X86ISD::AND, llvm::X86ISD::CMP, llvm::X86ISD::COMI, llvm::X86ISD::FCMP, llvm::X86ISD::OR, llvm::X86ISD::SBB, llvm::X86ISD::SMUL, llvm::X86ISD::SUB, llvm::X86ISD::UCOMI, llvm::X86ISD::UMUL, and llvm::X86ISD::XOR.

lower128BitShuffle()

static SDValue lower128BitShuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, MVT  VT, SDValue  V1, SDValue  V2, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Dispatching routine to lower various 128-bit x86 vector shuffles.

This routine breaks down the specific type of 128-bit shuffle and dispatches to the lowering routines accordingly.

Definition at line 14726 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getVectorShuffle(), llvm_unreachable, lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV4F32Shuffle(), lowerV4I32Shuffle(), lowerV8F16Shuffle(), lowerV8I16Shuffle(), and llvm::MVT::SimpleTy.

lower1BitShuffle()

static SDValue lower1BitShuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, MVT  VT, SDValue  V1, SDValue  V2, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 17591 of file X86ISelLowering.cpp.

References assert(), llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), CC, llvm::count_if(), llvm::APInt::countl_one(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, isBroadcastShuffleMask(), llvm::X86ISD::KSHIFTL, llvm::X86ISD::KSHIFTR, llvm_unreachable, lower1BitShuffleAsKSHIFTR(), match1BitShuffleAsKSHIFT(), llvm::Offset, llvm::ISD::SETCC, llvm::ISD::SETGT, llvm::ISD::SIGN_EXTEND, llvm::MVT::SimpleTy, llvm::ISD::TRUNCATE, llvm::X86Subtarget::useBWIRegs(), and widenMaskVector().

Referenced by lowerVECTOR_SHUFFLE().

lower1BitShuffleAsKSHIFTR()

static SDValue lower1BitShuffleAsKSHIFTR ( const SDLoc &  DL, ArrayRef< int >  Mask, MVT  VT, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 17516 of file X86ISelLowering.cpp.

References assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::X86ISD::KSHIFTR, llvm::SM_SentinelUndef, and widenMaskVector().

Referenced by lower1BitShuffle().

lower256BitShuffle()

static SDValue lower256BitShuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, MVT  VT, SDValue  V1, SDValue  V2, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

High-level routine to lower various 256-bit x86 vector shuffles.

This routine either breaks down the specific type of a 256-bit x86 vector shuffle or splits it into two 128-bit shuffles and fuses the results back together based on the available instructions.

Definition at line 16804 of file X86ISelLowering.cpp.

References llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::isInteger(), llvm_unreachable, lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsElementInsertion(), lowerShuffleWithUndefHalf(), lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV4I64Shuffle(), lowerV8F32Shuffle(), lowerV8I32Shuffle(), llvm::MVT::SimpleTy, and splitAndLowerShuffle().

lower512BitShuffle()

static SDValue lower512BitShuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, MVT  VT, SDValue  V1, SDValue  V2, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

High-level routine to lower various 512-bit x86 vector shuffles.

This routine either breaks down the specific type of a 512-bit x86 vector shuffle or splits it into two 256-bit shuffles and fuses the results back together based on the available instructions.

Definition at line 17442 of file X86ISelLowering.cpp.

References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX512(), llvm_unreachable, lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsBroadcast(), lowerShuffleAsElementInsertion(), lowerShuffleWithUndefHalf(), lowerV16F32Shuffle(), lowerV16I32Shuffle(), lowerV32I16Shuffle(), lowerV64I8Shuffle(), lowerV8F64Shuffle(), lowerV8I64Shuffle(), llvm::MVT::SimpleTy, and splitAndLowerShuffle().

Referenced by lowerVECTOR_SHUFFLE().

LowerABD()

static SDValue LowerABD ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 28984 of file X86ISelLowering.cpp.

References llvm::ISD::ABDS, llvm::MVT::bitsGE(), llvm::X86Subtarget::canUseCMOV(), CC, llvm::X86ISD::CMOV, llvm::X86::COND_B, llvm::X86::COND_L, llvm::SelectionDAG::getFreeze(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasInt256(), llvm::MVT::is256BitVector(), llvm::MVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), LHS, RHS, llvm::ISD::SIGN_EXTEND, splitVectorIntBinary(), llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, llvm::X86Subtarget::useBWIRegs(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerABS()

static SDValue LowerABS ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 28768 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::BLENDV, llvm::X86ISD::CMOV, llvm::X86::COND_NS, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is256BitVector(), llvm::MVT::isInteger(), splitVectorIntUnary(), and llvm::X86ISD::SUB.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerADDRSPACECAST()

static SDValue LowerADDRSPACECAST ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 32910 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getNode(), N, llvm::X86AS::PTR32_UPTR, llvm::report_fatal_error(), llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation(), and llvm::X86TargetLowering::ReplaceNodeResults().

LowerADDSAT_SUBSAT()

static SDValue LowerADDSAT_SUBSAT ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 28698 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::BitWidth, llvm::CallingConv::C, llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasInt256(), llvm::MVT::is256BitVector(), llvm::isConstOrConstSplat(), llvm::TargetLoweringBase::isOperationLegal(), llvm::MVT::isVector(), llvm::ISD::SADDO, llvm::ISD::SADDSAT, llvm::ISD::SETLT, llvm::ISD::SETUGT, splitVectorIntBinary(), llvm::ISD::SRA, llvm::ISD::SSUBO, llvm::ISD::SSUBSAT, llvm::ISD::SUB, llvm::ISD::UMAX, useVPTERNLOG(), llvm::ISD::USUBSAT, X, llvm::Xor, llvm::ISD::XOR, and Y.

Referenced by llvm::X86TargetLowering::LowerOperation().

lowerAddSub()

static SDValue lowerAddSub ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 28681 of file X86ISelLowering.cpp.

References assert(), DL, lowerAddSubToHorizontalOp(), and splitVectorIntBinary().

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerADDSUBO_CARRY()

static SDValue LowerADDSUBO_CARRY ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 32555 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADC, llvm::X86ISD::ADD, llvm::X86::COND_B, llvm::X86::COND_O, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getNode(), getSETCC(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::MERGE_VALUES, N, llvm::ISD::SADDO_CARRY, llvm::X86ISD::SBB, llvm::ISD::SSUBO_CARRY, llvm::ISD::TRUNCATE, and llvm::ISD::UADDO_CARRY.

Referenced by llvm::X86TargetLowering::LowerOperation().

lowerAddSubToHorizontalOp()

static SDValue lowerAddSubToHorizontalOp ( SDValue  Op, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation.

Definition at line 22042 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::BitWidth, DL, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::Value::hasOneUse(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, LHS, llvm_unreachable, RHS, shouldUseHorizontalOp(), llvm::ISD::SUB, std::swap(), and X.

Referenced by lowerAddSub().

LowerADJUST_TRAMPOLINE()

static SDValue LowerADJUST_TRAMPOLINE ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 27997 of file X86ISelLowering.cpp.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerAndToBT()

static SDValue LowerAndToBT ( SDValue  And, ISD::CondCode  CC, const SDLoc &  dl, SelectionDAG &  DAG, X86::CondCode &  X86CC  )

static

Result of 'and' is compared against zero.

Change to a BT node if possible. Returns the BT node and the condition code needed to use it.

Definition at line 23288 of file X86ISelLowering.cpp.

References llvm::And, llvm::ISD::AND, assert(), llvm::BitWidth, CC, llvm::SelectionDAG::computeKnownBits(), llvm::X86::COND_AE, llvm::X86::COND_B, llvm::ISD::Constant, llvm::KnownBits::countMinLeadingZeros(), getBT(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::ConstantSDNode::getZExtValue(), llvm::isBitwiseNot(), llvm::isOneConstant(), llvm::isPowerOf2_64(), llvm::Log2_64_Ceil(), llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::SHL, llvm::SelectionDAG::shouldOptForSize(), llvm::ISD::SRL, std::swap(), and llvm::ISD::TRUNCATE.

Referenced by combineAddOrSubToADCOrSBB().

LowerANY_EXTEND()

static SDValue LowerANY_EXTEND ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 24838 of file X86ISelLowering.cpp.

References assert(), DL, llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX(), LowerAVXExtend(), and LowerSIGN_EXTEND_Mask().

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerAsSplatVectorLoad()

static SDValue LowerAsSplatVectorLoad ( SDValue  SrcOp, MVT  VT, const SDLoc &  dl, SelectionDAG &  DAG  )

static

Definition at line 6937 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFrameInfo(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::InferPtrAlign(), llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::MachineFrameInfo::isFixedObjectIndex(), llvm::ISD::isNormalLoad(), llvm::Offset, Ptr, llvm::MachineFrameInfo::setObjectAlignment(), and llvm::Align::value().

LowerATOMIC_FENCE()

static SDValue LowerATOMIC_FENCE ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 31785 of file X86ISelLowering.cpp.

References emitLockedStackOp(), llvm::SelectionDAG::getNode(), llvm::X86Subtarget::hasMFence(), llvm::ISD::MEMBARRIER, llvm::X86ISD::MFENCE, llvm::SequentiallyConsistent, and llvm::SyncScope::System.

LowerATOMIC_STORE()

static SDValue LowerATOMIC_STORE ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 32473 of file X86ISelLowering.cpp.

References llvm::ISD::ATOMIC_SWAP, llvm::SelectionDAG::CreateStackTemporary(), emitLockedStackOp(), llvm::X86ISD::FILD, llvm::X86ISD::FIST, llvm::SelectionDAG::getAtomic(), llvm::SelectionDAG::getBitcast(), llvm::MachinePointerInfo::getFixedStack(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MachineMemOperand::MOLoad, llvm::MachineMemOperand::MOStore, llvm::ISD::SCALAR_TO_VECTOR, llvm::SequentiallyConsistent, and llvm::X86ISD::VEXTRACT_STORE.

Referenced by llvm::X86TargetLowering::LowerOperation().

lowerAtomicArith()

static SDValue lowerAtomicArith ( SDValue  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Lower atomic_load_ops into LOCK-prefixed operations.

Definition at line 32405 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_OR, llvm::ISD::ATOMIC_LOAD_SUB, llvm::ISD::ATOMIC_LOAD_XOR, DL, emitLockedStackOp(), llvm::SelectionDAG::getAtomic(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getSuccessOrdering(), llvm::MemSDNode::getSyncScopeID(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::isMinSignedConstant(), llvm::isNullConstant(), llvm::MemSDNode::isVolatile(), LHS, lowerAtomicArithWithLOCK(), llvm::ISD::MEMBARRIER, llvm::ISD::MERGE_VALUES, N, RHS, llvm::SequentiallyConsistent, and llvm::SyncScope::System.

Referenced by llvm::X86TargetLowering::LowerOperation().

lowerAtomicArithWithLOCK()

static SDValue lowerAtomicArithWithLOCK ( SDValue  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 32373 of file X86ISelLowering.cpp.

References llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::ATOMIC_LOAD_OR, llvm::ISD::ATOMIC_LOAD_SUB, llvm::ISD::ATOMIC_LOAD_XOR, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getVTList(), llvm::X86ISD::LADD, llvm::X86ISD::LAND, llvm_unreachable, llvm::X86ISD::LOR, llvm::X86ISD::LSUB, llvm::X86ISD::LXOR, and N.

Referenced by combineSetCCAtomicArith(), and lowerAtomicArith().

LowerAVG()

static SDValue LowerAVG ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 28804 of file X86ISelLowering.cpp.

References DL, llvm::X86Subtarget::hasInt256(), llvm::MVT::is256BitVector(), and splitVectorIntBinary().

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerAVXCONCAT_VECTORS()

static SDValue LowerAVXCONCAT_VECTORS ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 9377 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getFreeze(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::SDValue::hasOneUse(), llvm::Hi, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isFreezeUndef(), llvm::SDValue::isUndef(), llvm::Lo, and llvm::ArrayRef< T >::slice().

Referenced by LowerCONCAT_VECTORS().

LowerAVXExtend()

static SDValue LowerAVXExtend ( SDValue  Op, const SDLoc &  dl, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 20466 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), hasIdenticalHalvesShuffleMask(), llvm::X86Subtarget::hasInt256(), llvm::MVT::isVector(), splitVectorIntUnary(), and llvm::ISD::ZERO_EXTEND.

Referenced by LowerANY_EXTEND(), and LowerZERO_EXTEND().

LowerBITCAST()

static SDValue LowerBITCAST ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 31875 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getPMOVMSKB(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::Hi, llvm::MVT::isScalarInteger(), llvm::MVT::isVector(), llvm::Lo, llvm::X86ISD::MOVDQ2Q, llvm::ISD::SCALAR_TO_VECTOR, and llvm::SelectionDAG::SplitScalar().

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerBITREVERSE()

static SDValue LowerBITREVERSE ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 32230 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::BITREVERSE, llvm::ISD::BSWAP, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getGFNICtrlMask(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86ISD::GF2P8AFFINEQB, llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isVector(), llvm::Lo, LowerBITREVERSE_XOP(), Matrix, llvm::ISD::OR, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, splitVectorIntUnary(), and llvm::ISD::SRL.

Referenced by llvm::X86TargetLowering::LowerOperation(), and llvm::X86TargetLowering::ReplaceNodeResults().

LowerBITREVERSE_XOP()

static SDValue LowerBITREVERSE_XOP ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 32185 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITREVERSE, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, splitVectorIntUnary(), and llvm::X86ISD::VPPERM.

Referenced by LowerBITREVERSE().

LowerBUILD_VECTORAsVariablePermute()

static SDValue LowerBUILD_VECTORAsVariablePermute ( SDValue  V, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 8891 of file X86ISelLowering.cpp.

References createVariablePermute(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), Idx, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.

LowerBUILD_VECTORvXbf16()

static SDValue LowerBUILD_VECTORvXbf16 ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 7781 of file X86ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::MVT::changeVectorElementType(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

LowerBUILD_VECTORvXi1()

static SDValue LowerBUILD_VECTORvXi1 ( SDValue  Op, const SDLoc &  dl, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 7795 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::ISD::CONCAT_VECTORS, Cond, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::getSelect(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::Hi_32(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::Lo_32(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), Select, and llvm::ISD::SETCC.

lowerBuildVectorAsBroadcast()

static SDValue lowerBuildVectorAsBroadcast ( BuildVectorSDNode *  BVOp, const SDLoc &  dl, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which uses: a.

A single scalar load, or a constant. b. Repeated pattern of constants (e.g. <0,1,0,1> or <0,1,2,3,0,1,2,3>).

The VBROADCAST node is returned when a pattern is found, or SDValue() otherwise.

Definition at line 7441 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), llvm::ISD::BITCAST, llvm::CallingConv::C, llvm::ISD::Constant, llvm::ISD::ConstantFP, llvm::BitVector::count(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), getConstantVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::TargetLoweringBase::getPointerTy(), llvm::BuildVectorSDNode::getRepeatedSequence(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasNUsesOfValue(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::BuildVectorSDNode::isConstantSplat(), isFoldableUseOfShuffle(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::isNullConstantOrUndef(), llvm::SDNode::isOnlyUserOf(), llvm::EVT::isVector(), llvm::MachineMemOperand::MOLoad, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SelectionDAG::shouldOptForSize(), llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VBROADCASTM, and llvm::ISD::ZERO_EXTEND.

LowerBuildVectorAsInsert()

static SDValue LowerBuildVectorAsInsert ( SDValue  Op, const SDLoc &  DL, const APInt &  NonZeroMask, unsigned  NumNonZero, unsigned  NumZero, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 6661 of file X86ISelLowering.cpp.

References assert(), DL, llvm::First, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::INSERT_VECTOR_ELT, and llvm::ISD::SCALAR_TO_VECTOR.

Referenced by LowerBuildVectorv16i8(), and LowerBuildVectorv8i16().

lowerBuildVectorToBitOp()

static SDValue lowerBuildVectorToBitOp ( BuildVectorSDNode *  Op, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors.

NOTE: Its not in our interest to start make a general purpose vectorizer from this, but enough scalar bit operations are created from the later legalization + scalarization stages to need basic support.

Definition at line 8520 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::any_of(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::TargetLoweringBase::isOperationLegalOrPromote(), LHS, LowerShift(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and llvm::ISD::XOR.

LowerBuildVectorv16i8()

static SDValue LowerBuildVectorv16i8 ( SDValue  Op, const SDLoc &  DL, const APInt &  NonZeroMask, unsigned  NumNonZero, unsigned  NumZero, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Custom lower build_vector of v16i8.

Definition at line 6703 of file X86ISelLowering.cpp.

References assert(), DL, llvm::APInt::extractBits(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getZeroVector(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasSSE41(), I, llvm::ISD::INSERT_VECTOR_ELT, llvm::APInt::isZero(), LowerBuildVectorAsInsert(), llvm::ISD::OR, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SHL, llvm::ISD::TRUNCATE, and llvm::X86ISD::VZEXT_MOVL.

LowerBuildVectorv4x32()

static SDValue LowerBuildVectorv4x32 ( SDValue  Op, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Custom lower build_vector of v4i32 or v4f32.

Definition at line 6799 of file X86ISelLowering.cpp.

References assert(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), and llvm::X86ISD::MOVDDUP.

LowerBuildVectorv8i16()

static SDValue LowerBuildVectorv8i16 ( SDValue  Op, const SDLoc &  DL, const APInt &  NonZeroMask, unsigned  NumNonZero, unsigned  NumZero, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Custom lower build_vector of v8i16.

Definition at line 6785 of file X86ISelLowering.cpp.

References DL, llvm::X86Subtarget::hasSSE41(), and LowerBuildVectorAsInsert().

LowerCMP_SWAP()

static SDValue LowerCMP_SWAP ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 31808 of file X86ISelLowering.cpp.

References assert(), llvm::X86::COND_E, DL, llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), getSETCC(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::X86ISD::LCMPXCHG_DAG, llvm_unreachable, llvm::ISD::MERGE_VALUES, llvm::size(), and Success.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerCONCAT_VECTORS()

static SDValue LowerCONCAT_VECTORS ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 9519 of file X86ISelLowering.cpp.

References assert(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::MVT::getVectorElementType(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerAVXCONCAT_VECTORS(), and LowerCONCAT_VECTORSvXi1().

LowerCONCAT_VECTORSvXi1()

static SDValue LowerCONCAT_VECTORSvXi1 ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 9444 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::Hi, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::isPowerOf2_32(), llvm::isPowerOf2_64(), llvm::SDValue::isUndef(), llvm::X86ISD::KSHIFTL, llvm::Lo, llvm::Log2_64(), llvm::popcount(), llvm::ArrayRef< T >::slice(), widenMaskVectorType(), and widenSubVector().

Referenced by LowerCONCAT_VECTORS().

LowerCTLZ()

static SDValue LowerCTLZ ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 28617 of file X86ISelLowering.cpp.

References llvm::X86ISD::BSR, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::ISD::CTLZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), llvm::MVT::isVector(), LowerVectorCTLZ(), llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

LowerCTPOP()

static SDValue LowerCTPOP ( SDValue  N, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 32098 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::KnownBits::countMinTrailingZeros(), DL, llvm::KnownBits::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SelectionDAG::getShiftAmountOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), LowerVectorCTPOP(), LUT, llvm::ISD::MUL, N, llvm::ISD::SHL, llvm::ISD::SRL, and llvm::ISD::SUB.

LowerCTTZ()

static SDValue LowerCTTZ ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 28656 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::BSF, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::ISD::CTTZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::isKnownNeverZero(), and llvm::MVT::isVector().

LowerCVTPS2PH()

static SDValue LowerCVTPS2PH ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 32951 of file X86ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTPS2PH, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::Hi, llvm::Lo, and llvm::SelectionDAG::SplitVectorOperand().

Referenced by llvm::X86TargetLowering::LowerOperation(), and llvm::X86TargetLowering::ReplaceNodeResults().

LowerEXTEND_VECTOR_INREG()

static SDValue LowerEXTEND_VECTOR_INREG ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 24855 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::append(), assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::CONCAT_VECTORS, extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getFixedSizeInBits(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::Hi, I, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::Lo, llvm::ISD::SETGT, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::SM_SentinelUndef, llvm::X86ISD::VSRAI, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerEXTRACT_SUBVECTOR()

static SDValue LowerEXTRACT_SUBVECTOR ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 18763 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::X86ISD::KSHIFTR, and widenMaskVector().

LowerEXTRACT_VECTOR_ELT_SSE4()

static SDValue LowerEXTRACT_VECTOR_ELT_SSE4 ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 18194 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), Idx, llvm::MVT::is128BitVector(), llvm::isNullConstant(), llvm::X86::mayFoldIntoStore(), llvm::X86::mayFoldIntoZeroExtend(), llvm::X86ISD::PEXTRB, llvm::ISD::STORE, and llvm::ISD::TRUNCATE.

LowerFABSorFNEG()

static SDValue LowerFABSorFNEG ( SDValue  Op, SelectionDAG &  DAG  )

static

The only differences between FABS and FNEG are the mask and the logic op.

FNEG also has a folding opportunity for FNEG(FABS(x)).

Definition at line 22148 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FABS, llvm::X86ISD::FAND, llvm::ISD::FNEG, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SelectionDAG::getConstantFP(), llvm::MVT::getFltSemantics(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), and llvm::ISD::SCALAR_TO_VECTOR.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerFCanonicalize()

static SDValue LowerFCanonicalize ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 32976 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), N, and llvm::ISD::STRICT_FMUL.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerFCOPYSIGN()

static SDValue LowerFCOPYSIGN ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 22209 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::bitsGT(), llvm::MVT::bitsLT(), llvm::APFloat::clearSign(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getConstantFP(), llvm::MVT::getFltSemantics(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::isConstOrConstSplatFP(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), llvm::Or, and llvm::ISD::SCALAR_TO_VECTOR.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerFGETSIGN()

static SDValue LowerFGETSIGN ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 22278 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86ISD::MOVMSK, and llvm::ISD::SCALAR_TO_VECTOR.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerFMINIMUM_FMAXIMUM()

static SDValue LowerFMINIMUM_FMAXIMUM ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 28836 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::EVT::changeTypeToInteger(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FMAX, llvm::ISD::FMAXIMUM, llvm::X86ISD::FMIN, llvm::ISD::FMINIMUM, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), llvm::Hi, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::SelectionDAG::isKnownNeverNaN(), llvm::SelectionDAG::isKnownNeverZeroFloat(), llvm::EVT::isVector(), llvm::TargetOptions::NoNaNsFPMath, llvm::TargetOptions::NoSignedZerosFPMath, llvm::TargetMachine::Options, llvm::peekThroughBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SETLT, llvm::APInt::setSignBit(), llvm::ISD::SETUO, llvm::ISD::SPLAT_VECTOR, std::swap(), llvm::X86ISD::VFPCLASSS, X, and Y.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerFP16_TO_FP()

static SDValue LowerFP16_TO_FP ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 21957 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::CVTPH2PS, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::ISD::INSERT_VECTOR_ELT, and llvm::X86ISD::STRICT_CVTPH2PS.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerFP_TO_FP16()

static SDValue LowerFP_TO_FP16 ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 21986 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::CVTPS2PH, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValue(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::SCALAR_TO_VECTOR, and llvm::X86ISD::STRICT_CVTPS2PH.

Referenced by llvm::X86TargetLowering::LowerOperation().

lowerFPToIntToFP()

static SDValue lowerFPToIntToFP ( SDValue  CastToFP, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Given a scalar cast to FP with a cast to integer operand (almost an ftrunc), try to vectorize the cast ops.

This will avoid an expensive round-trip between XMM and GPR.

Definition at line 19478 of file X86ISelLowering.cpp.

References llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTTP2SI, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP_TO_SINT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SINT_TO_FP, and X.

LowerFROUND()

static SDValue LowerFROUND ( SDValue  Op, SelectionDAG &  DAG  )

static

ISD::FROUND is defined to round to nearest with ties rounding away from 0.

This mode isn't supported in hardware on X86. But as long as we aren't compiling with trapping math, we can emulate this with trunc(X + copysign(nextafter(0.5, 0.0), X)).

Definition at line 22126 of file X86ISelLowering.cpp.

References llvm::APFloat::convert(), llvm::ISD::FADD, llvm::ISD::FCOPYSIGN, llvm::ISD::FTRUNC, llvm::SelectionDAG::getConstantFP(), llvm::MVT::getFltSemantics(), llvm::SelectionDAG::getNode(), llvm::APFloat::next(), and llvm::APFloatBase::rmNearestTiesToEven.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerFSINCOS()

static SDValue LowerFSINCOS ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 32587 of file X86ISelLowering.cpp.

References assert(), llvm::CallingConv::C, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::StructType::get(), llvm::FixedVectorType::get(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getExternalSymbol(), llvm::SelectionDAG::getIntPtrConstant(), llvm::TargetLoweringBase::getLibcallName(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getTypeForEVT(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::isTargetDarwin(), llvm::TargetLowering::LowerCallTo(), llvm::ISD::MERGE_VALUES, RetTy, llvm::TargetLowering::CallLoweringInfo::setChain(), llvm::TargetLowering::CallLoweringInfo::setDebugLoc(), and llvm::TargetLowering::CallLoweringInfo::setLibCallee().

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerFunnelShift()

static SDValue LowerFunnelShift ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 30682 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), DL, llvm::ISD::FSHL, llvm::X86ISD::FSHL, llvm::ISD::FSHR, llvm::X86ISD::FSHR, llvm::SelectionDAG::getAnyExtOrTrunc(), getAVX512Node(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::MVT::getIntegerVT(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getPack(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SelectionDAG::getSplatSourceVector(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), getTargetVShiftNode(), getUnpackh(), getUnpackl(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::Hi, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::X86::isConstantSplat(), llvm::MVT::isVector(), llvm::Lo, llvm::ISD::OR, llvm::ISD::SHL, llvm::SelectionDAG::shouldOptForSize(), splitVectorOp(), llvm::ISD::SRL, supportedVectorShiftWithBaseAmnt(), supportedVectorShiftWithImm(), supportedVectorVarShift(), std::swap(), llvm::ISD::TRUNCATE, llvm::APInt::urem(), llvm::X86Subtarget::useBWIRegs(), useVPTERNLOG(), llvm::X86ISD::VSHLD, llvm::X86ISD::VSHLDV, llvm::X86ISD::VSHLI, llvm::X86ISD::VSHRD, llvm::X86ISD::VSHRDV, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.

LowerHorizontalByteSum()

static SDValue LowerHorizontalByteSum ( SDValue  V, MVT  VT, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Compute the horizontal sum of bytes in V for the elements of VT.

Requires V to be a byte vector and VT to be an integer vector type with wider elements than V's type. The width of the elements of VT determines how many bytes of V are summed horizontally to produce each element of the result.

Definition at line 31943 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), High, llvm::Low, llvm::X86ISD::PACKUS, llvm::X86ISD::PSADBW, llvm::ISD::SHL, and llvm::ISD::SRL.

Referenced by LowerVectorCTPOP().

LowerI64IntToFP16()

static SDValue LowerI64IntToFP16 ( SDValue  Op, const SDLoc &  dl, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 19376 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SINT_TO_FP, llvm::ISD::STRICT_SINT_TO_FP, llvm::ISD::STRICT_UINT_TO_FP, and llvm::ISD::UINT_TO_FP.

LowerI64IntToFP_AVX512DQ()

static SDValue LowerI64IntToFP_AVX512DQ ( SDValue  Op, const SDLoc &  dl, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 19334 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::MVT::getVectorVT(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SINT_TO_FP, llvm::ISD::STRICT_SINT_TO_FP, llvm::ISD::STRICT_UINT_TO_FP, and llvm::ISD::UINT_TO_FP.

LowerINSERT_SUBVECTOR()

static SDValue LowerINSERT_SUBVECTOR ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 18756 of file X86ISelLowering.cpp.

References assert(), and insert1BitVector().

Referenced by llvm::X86TargetLowering::LowerOperation().

lowerINT_TO_FP_vXi64()

static SDValue lowerINT_TO_FP_vXi64 ( SDValue  Op, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 19525 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::OR, llvm::ISD::SETLT, llvm::ISD::SINT_TO_FP, llvm::ISD::SRL, llvm::ISD::STRICT_FADD, llvm::ISD::STRICT_SINT_TO_FP, llvm::ISD::TokenFactor, and llvm::ISD::TRUNCATE.

Referenced by lowerUINT_TO_FP_vec().

LowerINTRINSIC_W_CHAIN()

static SDValue LowerINTRINSIC_W_CHAIN ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 27242 of file X86ISelLowering.cpp.

References llvm::X86ISD::AADD, llvm::X86ISD::AAND, Addr, llvm::X86ISD::AESDEC128KL, llvm::X86ISD::AESDEC256KL, llvm::X86ISD::AESDECWIDE128KL, llvm::X86ISD::AESDECWIDE256KL, llvm::X86ISD::AESENC128KL, llvm::X86ISD::AESENC256KL, llvm::X86ISD::AESENCWIDE128KL, llvm::X86ISD::AESENCWIDE256KL, llvm::X86ISD::AOR, assert(), llvm::X86ISD::AXOR, llvm::sampleprof::Base, CC, llvm::X86ISD::CMOV, llvm::X86ISD::CMPCCXADD, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::MachineFrameInfo::CreateStackObject(), DL, EmitMaskedTruncSStore(), EmitTruncSStore(), llvm::X86ISD::ENQCMD, llvm::X86ISD::ENQCMDS, expandIntrinsicWChainHelper(), llvm::GATHER, llvm::GATHER_AVX2, llvm::SDNode::getAsZExtVal(), getAVX2GatherNode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), getGatherNode(), llvm::MachineFunction::getInfo(), llvm::getIntrinsicWithChain(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getMaskedStore(), getMaskNode(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getPrefetchNode(), getReadTimeStampCounter(), llvm::SelectionDAG::getRegister(), llvm::MVT::getScalarSizeInBits(), getScatterNode(), getSETCC(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SelectionDAG::getTruncStore(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::INTR_TYPE_CAST_MMX, llvm::isAllOnesConstant(), llvm::X86::isExtendedSwiftAsyncFrameSupported(), isValid(), llvm::X86ISD::LADD, llvm::X86ISD::LAND, llvm::X86ISD::LBTC, llvm::X86ISD::LBTC_RM, llvm::X86ISD::LBTR, llvm::X86ISD::LBTR_RM, llvm::X86ISD::LBTS, llvm::X86ISD::LBTS_RM, llvm_unreachable, llvm::X86ISD::LOR, llvm::X86ISD::LSUB, llvm::X86ISD::LWPINS, llvm::X86ISD::LXOR, llvm::ManagedRA, MarkEHGuard(), MarkEHRegistrationNode(), llvm::ISD::MERGE_VALUES, llvm::Offset, llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, Operation, llvm::PREFETCH, llvm::X86ISD::RDPKRU, llvm::RDPMC, llvm::RDPRU, llvm::RDRAND, llvm::RDSEED, llvm::RDTSC, Results, llvm::SCATTER, llvm::MachineFrameInfo::setAdjustsStack(), llvm::X86MachineFunctionInfo::setAMXProgModel(), llvm::MachineFrameInfo::setFrameAddressIsTaken(), llvm::MachineFrameInfo::setHasCopyImplyingStackAdjustment(), llvm::ISD::SHL, Size, llvm::X86ISD::TESTUI, llvm::X86ISD::TPAUSE, llvm::TRUNCATE_TO_MEM_VI16, llvm::TRUNCATE_TO_MEM_VI32, llvm::TRUNCATE_TO_MEM_VI8, llvm::IntrinsicData::Type, llvm::X86ISD::UMWAIT, llvm::ISD::UNINDEXED, llvm::X86ISD::VTRUNC, llvm::X86ISD::VTRUNCS, llvm::X86ISD::VTRUNCUS, llvm::X86ISD::WRPKRU, llvm::XGETBV, llvm::XTEST, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerIntVSETCC_AVX512()

static SDValue LowerIntVSETCC_AVX512 ( SDValue  Op, const SDLoc &  dl, SelectionDAG &  DAG  )

static

Definition at line 23447 of file X86ISelLowering.cpp.

References assert(), CC, llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCSwappedOperands(), llvm::MVT::getVectorElementType(), llvm::ISD::SETLT, and std::swap().

Referenced by LowerVSETCC().

LowerLoad()

static SDValue LowerLoad ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 25186 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::isInteger(), and llvm::MVT::isVector().

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerMGATHER()

static SDValue LowerMGATHER ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 32857 of file X86ISelLowering.cpp.

References assert(), ExtendToType(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::is512BitVector(), llvm::SDValue::isUndef(), llvm::X86ISD::MGATHER, and N.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerMINMAX()

static SDValue LowerMINMAX ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 28820 of file X86ISelLowering.cpp.

References DL, llvm::X86Subtarget::hasInt256(), llvm::MVT::is256BitVector(), and splitVectorIntBinary().

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerMLOAD()

static SDValue LowerMLOAD ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 32749 of file X86ISelLowering.cpp.

References assert(), ExtendToType(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), N, Select, and llvm::ISD::VSELECT.

LowerMSCATTER()

static SDValue LowerMSCATTER ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 32686 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, ExtendToType(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTypeToTransformTo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::is512BitVector(), llvm::X86ISD::MSCATTER, and N.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerMSTORE()

static SDValue LowerMSTORE ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 32815 of file X86ISelLowering.cpp.

References assert(), ExtendToType(), llvm::SelectionDAG::getMaskedStore(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::is512BitVector(), and N.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerMUL()

static SDValue LowerMUL ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 29037 of file X86ISelLowering.cpp.

References A, llvm::ISD::ADD, llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::ISD::ANY_EXTEND, assert(), B, llvm::X86Subtarget::canExtendTo512BW(), llvm::SelectionDAG::computeKnownBits(), llvm::enumerate(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), getPack(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, Idx, llvm::MVT::is256BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isNullConstantOrUndef(), llvm::APInt::isSubsetOf(), llvm::ISD::MUL, llvm::ISD::OR, llvm::X86ISD::PMULUDQ, llvm::SmallVectorTemplateBase< T, bool >::push_back(), splitVectorIntBinary(), llvm::ISD::TRUNCATE, llvm::X86ISD::VPMADDUBSW, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLI, and llvm::KnownBits::Zero.

LowerMULH()

static SDValue LowerMULH ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 29288 of file X86ISelLowering.cpp.

References A, llvm::ISD::ADD, llvm::ISD::AND, assert(), B, llvm::X86Subtarget::canExtendTo512BW(), Fixup, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), getTargetVShiftByConstNode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is256BitVector(), LowervXi8MulWithUNPCK(), llvm::Mul, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::X86ISD::PMULDQ, llvm::X86ISD::PMULUDQ, llvm::ISD::SETGT, llvm::ISD::SIGN_EXTEND, splitVectorIntBinary(), llvm::ISD::SUB, T1, llvm::ISD::TRUNCATE, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerMULO()

static SDValue LowerMULO ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 29395 of file X86ISelLowering.cpp.

References A, B, llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftByConstNode(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasInt256(), llvm::Hi, High, llvm::MVT::isVector(), llvm::Lo, llvm::Low, LowervXi8MulWithUNPCK(), LowerXALUO(), llvm::Mul, llvm::ISD::MUL, llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMULO, splitVector(), llvm::ISD::SRA, llvm::ISD::TRUNCATE, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerPARITY()

static SDValue LowerPARITY ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 32317 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::X86ISD::CMP, llvm::X86::COND_NP, DL, llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getSETCC(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::Lo, llvm::SelectionDAG::MaskedValueIsZero(), llvm::ISD::SRL, llvm::ISD::TRUNCATE, X, llvm::ISD::XOR, llvm::X86ISD::XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerPREFETCH()

static SDValue LowerPREFETCH ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 32964 of file X86ISelLowering.cpp.

LowerREADCYCLECOUNTER()

static SDValue LowerREADCYCLECOUNTER ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 27165 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getMergeValues(), getReadTimeStampCounter(), and Results.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerRotate()

static SDValue LowerRotate ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 30884 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::X86ISD::BLENDV, llvm::CallingConv::C, convertShiftLeftToScale(), DL, llvm::SelectionDAG::FoldConstantArithmetic(), llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getGFNICtrlMask(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getPack(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SelectionDAG::getSplatSourceVector(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftByConstNode(), getTargetVShiftNode(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86ISD::GF2P8AFFINEQB, llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::X86::isConstantSplat(), llvm::SelectionDAG::isSplatValue(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), llvm::Lo, llvm::ISD::MUL, llvm::ISD::MULHU, llvm::ISD::OR, llvm::X86ISD::PCMPGT, llvm::X86ISD::PMULUDQ, llvm::ISD::ROTL, llvm::ISD::SHL, splitVectorIntBinary(), llvm::ISD::SRL, llvm::ISD::SUB, supportedVectorShiftWithImm(), supportedVectorVarShift(), llvm::ISD::TRUNCATE, llvm::APInt::urem(), llvm::X86Subtarget::useBWIRegs(), useVPTERNLOG(), llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerSCALAR_TO_VECTOR()

static SDValue LowerSCALAR_TO_VECTOR ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 18717 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), insert128BitVector(), llvm::MVT::is128BitVector(), llvm::MVT::isInteger(), llvm::X86::isZeroNode(), and llvm::ISD::SCALAR_TO_VECTOR.

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerSELECTWithCmpZero()

static SDValue LowerSELECTWithCmpZero ( SDValue  CmpVal, SDValue  LHS, SDValue  RHS, unsigned  X86CC, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 24389 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::And, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::EVT::bitsGT(), llvm::EVT::bitsLT(), llvm::X86Subtarget::canUseCMOV(), llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::isAllOnesConstant(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::EVT::isScalarInteger(), LHS, llvm::Or, llvm::ISD::OR, RHS, llvm::X86ISD::SETCC_CARRY, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and Y.

LowerShift()

static SDValue LowerShift ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 30007 of file X86ISelLowering.cpp.

References A, llvm::ISD::ADD, llvm::ISD::AND, llvm::APInt::ashr(), assert(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::begin(), llvm::X86ISD::BLENDV, llvm::CallingConv::C, llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), canWidenShuffleElements(), llvm::SelectionDAG::computeKnownBits(), convertShiftLeftToScale(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::count(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::empty(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitsSet(), llvm::SelectionDAG::getBuildVector(), llvm::KnownBits::getConstant(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SelectionDAG::getExtOrTrunc(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNegative(), llvm::SDValue::getNode(), getNode(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::APInt::getSignMask(), llvm::MVT::getSizeInBits(), llvm::APInt::getSplat(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, I, is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::KnownBits::isConstant(), llvm::X86::isConstantSplat(), llvm::SelectionDAG::isKnownNeverZero(), isUndef(), llvm::MVT::isVector(), llvm::Lo, LowerShiftByScalarImmediate(), LowerShiftByScalarVariable(), llvm::APInt::lshr(), LUT, llvm::Masked, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::ISD::OR, llvm::X86ISD::PACKUS, llvm::X86ISD::PCMPGT, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), R2, llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SETEQ, llvm::ISD::SETGT, llvm::APInt::shl(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, splitVectorIntBinary(), llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, supportedVectorVarShift(), std::swap(), llvm::ISD::TRUNCATE, llvm::X86ISD::VPSHA, llvm::X86ISD::VPSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, WidenShift, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.

LowerShiftByScalarImmediate()

static SDValue LowerShiftByScalarImmediate ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 29727 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getFreeze(), getGFNICtrlMask(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::APInt::getZExtValue(), llvm::X86ISD::GF2P8AFFINEQB, llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE42(), llvm::MVT::is512BitVector(), llvm::X86::isConstantSplat(), llvm_unreachable, llvm::Lower, llvm::X86ISD::PCMPGT, llvm::ISD::SETGT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, supportedVectorShiftWithImm(), llvm::APInt::uge(), llvm::Upper, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and llvm::ISD::XOR.

Referenced by LowerShift().

LowerShiftByScalarVariable()

static SDValue LowerShiftByScalarVariable ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 29888 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSplatSourceVector(), getTargetVShiftByConstNode(), getTargetVShiftNode(), getTargetVShiftUniformOpcode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, supportedVectorShiftWithBaseAmnt(), and llvm::ISD::XOR.

Referenced by LowerShift().

LowerShiftParts()

static SDValue LowerShiftParts ( SDValue  Op, SelectionDAG &  DAG  )

static

Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount.

TODO: Can this be moved to general expansion code?

Definition at line 19326 of file X86ISelLowering.cpp.

References llvm::TargetLowering::expandShiftParts(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Hi, and llvm::Lo.

Referenced by llvm::X86TargetLowering::LowerOperation().

lowerShuffleAsBitBlend()

static SDValue lowerShuffleAsBitBlend ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, SelectionDAG &  DAG  )

static

Try to emit a blend instruction for a shuffle using bit math.

This is used as a fallback approach when first class blend instructions are unavailable. Currently it is only suitable for integer vectors, but could be generalized for floating point vectors if desirable.

Definition at line 10738 of file X86ISelLowering.cpp.

References llvm::AllOnes, assert(), DL, llvm::SelectionDAG::getAllOnesConstant(), getBitSelect(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getVectorElementType(), llvm::MVT::isInteger(), and Size.

Referenced by lower256BitShuffle(), lower512BitShuffle(), lowerShuffleAsBlend(), lowerV16I8Shuffle(), and lowerV8I16Shuffle().

lowerShuffleAsBitMask()

static SDValue lowerShuffleAsBitMask ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to emit a bitmask instruction for a shuffle.

This handles cases where we can model a blend exactly as a bitmask due to one of the inputs being zeroable.

Definition at line 10683 of file X86ISelLowering.cpp.

References llvm::AllOnes, llvm::And, llvm::ISD::AND, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::APFloat::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getFltSemantics(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), and Size.

Referenced by lower256BitShuffle(), lower512BitShuffle(), lowerShuffleAsBlend(), lowerV16I8Shuffle(), lowerV4I32Shuffle(), lowerV64I8Shuffle(), and lowerV8I16Shuffle().

lowerShuffleAsBitRotate()

static SDValue lowerShuffleAsBitRotate ( const SDLoc &  DL, MVT  VT, SDValue  V1, ArrayRef< int >  Mask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Lower shuffle using X86ISD::VROTLI rotations.

Definition at line 11497 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE3(), llvm::MVT::is128BitVector(), matchShuffleAsBitRotate(), llvm::ISD::OR, llvm::X86ISD::VROTLI, llvm::X86ISD::VSHLI, and llvm::X86ISD::VSRLI.

Referenced by lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4I32Shuffle(), lowerV64I8Shuffle(), lowerV8I16Shuffle(), and lowerV8I32Shuffle().

lowerShuffleAsBlend()

static SDValue lowerShuffleAsBlend ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Original, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to emit a blend instruction for a shuffle.

This doesn't do any checks for the availability of instructions for blending these values. It relies on the availability of the X86ISD::BLENDI pattern to be matched in the backend with the type given. What it does check for is that the shuffle mask is a blend, or convertible into a blend with zero.

Definition at line 10845 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::BLENDI, llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSignedConstant(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), getVectorMaskingNode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, is128BitLaneRepeatedShuffleMask(), llvm::ISD::isNormalLoad(), llvm_unreachable, llvm::Lo, lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), llvm::Masked, matchShuffleAsBlend(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::shouldOptForSize(), llvm::MVT::SimpleTy, llvm::SmallVectorBase< Size_T >::size(), Size, and std::swap().

Referenced by lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV2X128Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().

lowerShuffleAsBlendAndPermute()

static SDValue lowerShuffleAsBlendAndPermute ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, SelectionDAG &  DAG, bool  ImmBlends = false  )

static

Try to lower as a blend of elements from two inputs followed by a single-input permutation.

This matches the pattern where we can blend elements from two inputs and then reduce the shuffle to a single-input permutation.

Definition at line 11006 of file X86ISelLowering.cpp.

References assert(), canWidenShuffleElements(), DL, llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), and Size.

Referenced by lowerShuffleAsDecomposedShuffleMerge(), lowerV4F32Shuffle(), and lowerV4I64Shuffle().

lowerShuffleAsBlendOfPSHUFBs()

static SDValue lowerShuffleAsBlendOfPSHUFBs ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, SelectionDAG &  DAG, bool &  V1InUse, bool &  V2InUse  )

static

Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used.

Definition at line 14022 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorVT(), is128BitLaneCrossingShuffleMask(), llvm::ISD::OR, llvm::X86ISD::PSHUFB, and Size.

Referenced by lowerV16I8Shuffle(), lowerV64I8Shuffle(), and lowerV8I16Shuffle().

lowerShuffleAsBroadcast()

static SDValue lowerShuffleAsBroadcast ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to lower broadcast of a single element.

For convenience, this code also bundles all of the subtarget feature set filtering. While a little annoying to re-dispatch on type here, there isn't a convenient way to factor it out.

Definition at line 12689 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::count_if(), DL, extract128BitVector(), llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::getSplatIndex(), llvm::MVT::getStoreSize(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE3(), Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isInteger(), llvm::ISD::isNormalLoad(), isShuffleFoldableLoad(), lowerShuffleAsTruncBroadcast(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::X86ISD::MOVDDUP, llvm::Offset, llvm::peekThroughBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SM_SentinelUndef, llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VBROADCAST_LOAD.

Referenced by lower512BitShuffle(), lowerV16I16Shuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV8F16Shuffle(), lowerV8F32Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerVECTOR_SHUFFLE().

lowerShuffleAsByteRotate()

static SDValue lowerShuffleAsByteRotate ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 11648 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is512BitVector(), isNoopShuffleMask(), llvm::Lo, matchShuffleAsByteRotate(), llvm::ISD::OR, llvm::X86ISD::PALIGNR, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.

Referenced by lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV2I64Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().

lowerShuffleAsByteRotateAndPermute()

static SDValue lowerShuffleAsByteRotateAndPermute ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then permuting the elements of the result in place.

Definition at line 11239 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, is128BitLaneCrossingShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::Lo, llvm::X86ISD::PALIGNR, and llvm::SM_SentinelUndef.

Referenced by lowerShuffleAsDecomposedShuffleMerge(), lowerV16I8Shuffle(), and lowerV64I8Shuffle().

lowerShuffleAsByteShiftMask()

static SDValue lowerShuffleAsByteShiftMask ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to lower a vector shuffle as a byte shift sequence.

Definition at line 11756 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::countl_one(), llvm::APInt::countr_one(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::is128BitVector(), isNoopShuffleMask(), isSequentialOrUndefInRange(), isUndefOrInRange(), llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.

Referenced by lowerV16I8Shuffle(), and lowerV8I16Shuffle().

lowerShuffleAsDecomposedShuffleMerge()

static SDValue lowerShuffleAsDecomposedShuffleMerge ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Generic routine to decompose a shuffle and blend into independent blends and permutes.

This matches the extremely common pattern for handling combined shuffle+blend operations on newer X86 ISAs where we have very fast blend operations. It will try to pick the best arrangement of shuffles and blends. For vXi8/vXi16 shuffles we may use unpack instead of blend.

Definition at line 11356 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), DL, llvm::enumerate(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), I, isBroadcastShuffleMask(), isNoopOrBroadcastShuffleMask(), isNoopShuffleMask(), isSingleElementRepeatedMask(), lowerShuffleAsBlendAndPermute(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsUNPCKAndPermute(), llvm::X86::mayFoldLoad(), and llvm::X86ISD::VBROADCAST.

Referenced by lowerShuffleAsSplitOrBlend(), lowerV16I8Shuffle(), lowerV2I64Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV8F32Shuffle(), lowerV8I16Shuffle(), and lowerV8I32Shuffle().

lowerShuffleAsElementInsertion()

static SDValue lowerShuffleAsElementInsertion ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to lower insertion of a single element into a zero vector.

This is a common pattern that we have especially efficient patterns to lower across all subtarget feature sets.

Definition at line 12420 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), DL, llvm::find_if(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), getScalarValueForVectorElement(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), llvm::MVT::is128BitVector(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), isSoftF16(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSS, llvm::ISD::OR, llvm::ISD::SCALAR_TO_VECTOR, Size, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VZEXT_MOVL, and llvm::ISD::ZERO_EXTEND.

Referenced by lower256BitShuffle(), lower512BitShuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV4F32Shuffle(), lowerV4I32Shuffle(), lowerV8F16Shuffle(), and lowerV8I16Shuffle().

lowerShuffleAsInsertPS()

static SDValue lowerShuffleAsInsertPS ( const SDLoc &  DL, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, SelectionDAG &  DAG  )

static

Definition at line 12984 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::X86ISD::INSERTPS, and matchShuffleAsInsertPS().

Referenced by lowerV4F32Shuffle().

lowerShuffleAsLanePermuteAndPermute()

static SDValue lowerShuffleAsLanePermuteAndPermute ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane permutation.

This is mainly for cases where we can have non-repeating permutes in each lane.

TODO: This is very similar to lowerShuffleAsLanePermuteAndRepeatedMask, we should investigate merging them.

Attempts to find a sublane permute with the given size that gets all elements into their target lanes.

If successful, fills CrossLaneMask and InLaneMask and returns true. If unsuccessful, returns false and may overwrite InLaneMask.

Definition at line 14984 of file X86ISelLowering.cpp.

References DL, llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), isSequentialOrUndefInRange(), isUndefOrEqual(), llvm::narrowShuffleMaskElts(), llvm::APInt::setBit(), and llvm::SM_SentinelUndef.

Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), and lowerV64I8Shuffle().

lowerShuffleAsLanePermuteAndRepeatedMask()

static SDValue lowerShuffleAsLanePermuteAndRepeatedMask ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.

This attempts to create a repeated lane shuffle where each lane uses one or two of the lanes of the inputs. The lanes of the input vectors are shuffled in one or two independent shuffles to get the lanes into the position needed by the final shuffle.

Definition at line 15294 of file X86ISelLowering.cpp.

References assert(), llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getVectorShuffle(), is128BitLaneRepeatedShuffleMask(), llvm::M1(), and std::swap().

Referenced by lowerV16I16Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().

lowerShuffleAsLanePermuteAndShuffle()

static SDValue lowerShuffleAsLanePermuteAndShuffle ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one source with a lane permutation.

This lowering strategy results in four instructions in the worst case for a single-input cross lane shuffle which is lower than any other fully general cross-lane shuffle strategy I'm aware of. Special cases for each particular shuffle pattern should be handled prior to trying this lowering.

Definition at line 15115 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), computeInLaneShuffleMask(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::MVT::isFloatingPoint(), lowerShuffleAsLanePermuteAndSHUFP(), Size, and splitAndLowerShuffle().

Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), and lowerV8F32Shuffle().

lowerShuffleAsLanePermuteAndSHUFP()

static SDValue lowerShuffleAsLanePermuteAndSHUFP ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, SelectionDAG &  DAG  )

static

Definition at line 14948 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), getSHUFPDImmForMask(), llvm::SelectionDAG::getVectorShuffle(), LHS, RHS, and llvm::X86ISD::SHUFP.

Referenced by lowerShuffleAsLanePermuteAndShuffle(), and lowerV4F64Shuffle().

lowerShuffleAsPermuteAndUnpack()

static SDValue lowerShuffleAsPermuteAndUnpack ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.

This specifically targets cases where we end up with alternating between the two inputs, and so can permute them into something that feeds a single UNPCK instruction. Note that this routine only targets integer vectors because for floating point vectors we have a generalized SHUFPS lowering strategy that handles everything that doesn't exactly match an unpack, making this clever lowering unnecessary.

Definition at line 11128 of file X86ISelLowering.cpp.

References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), Size, llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by lowerShuffleAsDecomposedShuffleMerge(), lowerV16I8Shuffle(), lowerV4I32Shuffle(), and lowerV8I16Shuffle().

lowerShuffleAsRepeatedMaskAndLanePermute()

static SDValue lowerShuffleAsRepeatedMaskAndLanePermute ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes.

Definition at line 15669 of file X86ISelLowering.cpp.

References assert(), DL, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), Idx, is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), isUndefOrInRange(), llvm::M1(), and llvm::SM_SentinelUndef.

Referenced by lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().

lowerShuffleAsShift()

static SDValue lowerShuffleAsShift ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, bool  BitwiseOnly  )

static

Definition at line 11905 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::TargetLoweringBase::isTypeLegal(), matchShuffleAsShift(), Size, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.

Referenced by lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV2I64Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().

lowerShuffleAsSpecificZeroOrAnyExtend()

static SDValue lowerShuffleAsSpecificZeroOrAnyExtend ( const SDLoc &  DL, MVT  VT, int  Scale, int  Offset, bool  AnyExt, SDValue  InputV, ArrayRef< int >  Mask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Lower a vector shuffle as a zero or any extension.

Given a specific number of elements, element bit width, and extension stride, produce either a zero or any extension based on the available features of the subtarget. The extended elements are consecutive and begin and can start from an offsetted element index in the input; to avoid excess shuffling the offset must either being in the bottom lane or at the start of a higher lane. All extended elements must be from the same lane.

Definition at line 12093 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), DL, llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getEXTEND_VECTOR_INREG(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, Idx, llvm::MVT::is128BitVector(), isUndefUpperHalf(), llvm::Lo, llvm::Offset, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::ISD::ZERO_EXTEND.

Referenced by lowerShuffleAsZeroOrAnyExtend().

lowerShuffleAsSplitOrBlend()

static SDValue lowerShuffleAsSplitOrBlend ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Either split a vector in halves or decompose the shuffles and the blend/unpack.

This is provided as a good fallback for many lowerings of non-single-input shuffles with more than one 128-bit lane. In those cases, we want to select between splitting the shuffle into 128-bit components and stitching those back together vs. extracting the single-input shuffles and blending those results.

Definition at line 14893 of file X86ISelLowering.cpp.

References assert(), llvm::SmallBitVector::count(), llvm::count(), DL, llvm::MVT::getSizeInBits(), lowerShuffleAsDecomposedShuffleMerge(), llvm::SmallBitVector::resize(), Size, and splitAndLowerShuffle().

Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().

lowerShuffleAsTruncBroadcast()

static SDValue lowerShuffleAsTruncBroadcast ( const SDLoc &  DL, MVT  VT, SDValue  V0, int  BroadcastIdx, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements.

This assumes we have AVX2.

Definition at line 12546 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::isInteger(), llvm::MVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::X86ISD::VBROADCAST.

Referenced by lowerShuffleAsBroadcast().

lowerShuffleAsUNPCKAndPermute()

static SDValue lowerShuffleAsUNPCKAndPermute ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, SelectionDAG &  DAG  )

static

Try to lower as an unpack of elements from two inputs followed by a single-input permutation.

This matches the pattern where we can unpack elements from two inputs and then reduce the shuffle to a single-input (wider) permutation.

Definition at line 11045 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::Hi, isUndefOrInRange(), llvm::Lo, llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by lowerShuffleAsDecomposedShuffleMerge().

lowerShuffleAsVALIGN()

static SDValue lowerShuffleAsVALIGN ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to lower a vector shuffle as a dword/qword rotation.

AVX512 has a VALIGND/VALIGNQ instructions that will do an arbitrary rotation of the concatenation of two vectors; This routine will try to generically lower a vector shuffle through such an pattern.

Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.

Definition at line 11705 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::countl_one(), llvm::APInt::countr_one(), DL, llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getTargetConstant(), getZeroVector(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), isSequentialOrUndefInRange(), llvm::Lo, llvm::Low, matchShuffleAsElementRotate(), and llvm::X86ISD::VALIGN.

Referenced by lowerV16I32Shuffle(), lowerV2I64Shuffle(), lowerV4I64Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().

lowerShuffleAsVTRUNC()

static SDValue lowerShuffleAsVTRUNC ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 10408 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::extractBits(), getAVX512TruncNode(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::APInt::isAllOnes(), llvm::ISD::isNormalLoad(), isSequentialOrUndefInRange(), isUndefInRange(), llvm::Lo, llvm::Offset, llvm::peekThroughBitcasts(), and llvm::X86ISD::VSRLI.

Referenced by lowerV16I16Shuffle(), lowerV16I8Shuffle(), lowerV32I8Shuffle(), and lowerV8I16Shuffle().

lowerShuffleAsVTRUNCAndUnpack()

static SDValue lowerShuffleAsVTRUNCAndUnpack ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, SelectionDAG &  DAG  )

static

Definition at line 15941 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::countl_one(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::INSERT_SUBVECTOR, isSequentialOrUndefInRange(), and llvm::X86ISD::VTRUNC.

Referenced by lowerV32I8Shuffle().

lowerShuffleAsZeroOrAnyExtend()

static SDValue lowerShuffleAsZeroOrAnyExtend ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to lower a vector shuffle as a zero extension on any microarch.

This routine will try to do everything in its power to cleverly lower a shuffle which happens to match the pattern of a zero extend. It doesn't check for the profitability of this lowering, it tries to aggressively match this pattern. It will use all of the micro-architectural details it can to emit an efficient lowering. It handles both blends with all-zero inputs to explicitly zero-extend and undef-lanes (sometimes undef due to masking out later).

The reason we have dedicated lowering for zext-style shuffles is that they are both incredibly common and often quite performance sensitive.

Definition at line 12257 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), isSequentialOrUndefInRange(), llvm::Lower, lowerShuffleAsSpecificZeroOrAnyExtend(), llvm::Offset, and llvm::X86ISD::VZEXT_MOVL.

Referenced by lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4I32Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), lowerV8I16Shuffle(), and lowerV8I32Shuffle().

lowerShuffleOfExtractsAsVperm()

static SDValue lowerShuffleOfExtractsAsVperm ( const SDLoc &  DL, SDValue  N0, SDValue  N1, ArrayRef< int >  Mask, SelectionDAG &  DAG  )

static

If we are extracting two 128-bit halves of a vector and shuffling the result, match that to a 256-bit AVX2 vperm* instruction to avoid a multi-shuffle lowering.

Definition at line 12635 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::append(), assert(), llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::hasOneUse(), is128BitUnpackShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), and isSingleSHUFPSMask().

Referenced by lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV4F32Shuffle(), and lowerV4I32Shuffle().

lowerShufflePairAsUNPCKAndPermute()

static SDValue lowerShufflePairAsUNPCKAndPermute ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, SelectionDAG &  DAG  )

static

Definition at line 15989 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), I, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::SmallVectorBase< Size_T >::size(), Size, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::SDNode::users(), llvm::ISD::VECTOR_SHUFFLE, and llvm::X86ISD::VPERM2X128.

Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().

lowerShuffleWithEXPAND()

static SDValue lowerShuffleWithEXPAND ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 10094 of file X86ISelLowering.cpp.

References assert(), DL, llvm::X86ISD::EXPAND, llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), getMaskNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), and isNonZeroElementsInOrder().

Referenced by lowerV16F32Shuffle(), lowerV16I32Shuffle(), lowerV4F64Shuffle(), lowerV4I64Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().

lowerShuffleWithPACK()

static SDValue lowerShuffleWithPACK ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 10633 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::Log2_32(), matchShuffleWithPACK(), and llvm::X86ISD::PACKSS.

Referenced by lowerV16I16Shuffle(), lowerV16I8Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV64I8Shuffle(), and lowerV8I16Shuffle().

lowerShuffleWithPERMV()

static SDValue lowerShuffleWithPERMV ( const SDLoc &  DL, MVT  VT, ArrayRef< int >  OriginalMask, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 14320 of file X86ISelLowering.cpp.

References llvm::MVT::changeTypeToInteger(), llvm::ShuffleVectorSDNode::commuteMask(), DL, extractSubVector(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::is512BitVector(), isShuffleFoldableLoad(), std::swap(), llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, and widenSubVector().

Referenced by combineX86ShuffleChain(), lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV64I8Shuffle(), lowerV8F64Shuffle(), and lowerV8I64Shuffle().

lowerShuffleWithPSHUFB()

static SDValue lowerShuffleWithPSHUFB ( const SDLoc &  DL, MVT  VT, ArrayRef< int >  Mask, SDValue  V1, SDValue  V2, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to lower a shuffle with a single PSHUFB of V1 or V2.

Definition at line 10036 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::X86ISD::PSHUFB, and Size.

Referenced by lowerV16I16Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), and lowerV64I8Shuffle().

lowerShuffleWithSHUFPD()

static SDValue lowerShuffleWithSHUFPD ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 15914 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), getZeroVector(), matchShuffleWithSHUFPD(), and llvm::X86ISD::SHUFP.

Referenced by lowerV4F64Shuffle(), and lowerV8F64Shuffle().

lowerShuffleWithSHUFPS()

static SDValue lowerShuffleWithSHUFPS ( const SDLoc &  DL, MVT  VT, ArrayRef< int >  Mask, SDValue  V1, SDValue  V2, SelectionDAG &  DAG  )

static

Lower a vector shuffle using the SHUFPS instruction.

This is a helper routine dedicated to lowering vector shuffles using SHUFPS. It makes no assumptions about whether this is the best lowering, it simply uses it.

Definition at line 13191 of file X86ISelLowering.cpp.

References llvm::ShuffleVectorSDNode::commuteMask(), llvm::count_if(), DL, llvm::find_if(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), lowerShuffleWithSHUFPS(), llvm::X86ISD::SHUFP, and std::swap().

Referenced by lowerShuffleWithSHUFPS(), lowerV16F32Shuffle(), lowerV16I32Shuffle(), lowerV4F32Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().

lowerShuffleWithSSE4A()

static SDValue lowerShuffleWithSSE4A ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, SelectionDAG &  DAG  )

static

Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.

Definition at line 12066 of file X86ISelLowering.cpp.

References DL, llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::X86ISD::INSERTQI, matchShuffleAsEXTRQ(), and matchShuffleAsINSERTQ().

Referenced by lowerV16I8Shuffle(), and lowerV8I16Shuffle().

lowerShuffleWithUndefHalf()

static SDValue lowerShuffleWithUndefHalf ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.

This allows for fast cases such as subvector extraction/insertion or shuffling smaller vector types which can lower more efficiently.

Definition at line 15558 of file X86ISelLowering.cpp.

References assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::MVT::getHalfNumVectorElementsVT(), getHalfShuffleMask(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getShuffleHalfVectors(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::Hi, llvm::ISD::INSERT_SUBVECTOR, is128BitUnpackShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isSequentialOrUndefInRange(), isSingleSHUFPSMask(), isUndefLowerHalf(), isUndefUpperHalf(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by lower256BitShuffle(), and lower512BitShuffle().

lowerShuffleWithUNPCK()

static SDValue lowerShuffleWithUNPCK ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, SelectionDAG &  DAG  )

static

Definition at line 10206 of file X86ISelLowering.cpp.

References llvm::ShuffleVectorSDNode::commuteMask(), llvm::createUnpackShuffleMask(), DL, llvm::SelectionDAG::getNode(), isShuffleEquivalent(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().

lowerShuffleWithUNPCK256()

static SDValue lowerShuffleWithUNPCK256 ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, SelectionDAG &  DAG  )

static

Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit) followed by unpack 256-bit.

Definition at line 10233 of file X86ISelLowering.cpp.

References llvm::createSplat2ShuffleMask(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), isShuffleEquivalent(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), and lowerV8I32Shuffle().

lowerShuffleWithVPMOV()

static SDValue lowerShuffleWithVPMOV ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 10358 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::APInt::extractBits(), getAVX512TruncNode(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::APInt::isAllOnes(), isSequentialOrUndefInRange(), isUndefInRange(), llvm::peekThroughBitcasts(), and llvm::ISD::TRUNCATE.

Referenced by lowerV16I8Shuffle(), and lowerV8I16Shuffle().

LowerSIGN_EXTEND()

static SDValue LowerSIGN_EXTEND ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 24978 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasInt256(), llvm::MVT::isVector(), LowerSIGN_EXTEND_Mask(), llvm::ISD::SIGN_EXTEND_VECTOR_INREG, and splitVectorIntUnary().

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerSIGN_EXTEND_Mask()

static SDValue LowerSIGN_EXTEND_Mask ( SDValue  Op, const SDLoc &  dl, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 24783 of file X86ISelLowering.cpp.

References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is512BitVector(), SplitAndExtendv16i1(), and llvm::ISD::TRUNCATE.

Referenced by LowerANY_EXTEND(), and LowerSIGN_EXTEND().

LowerStore()

static SDValue LowerStore ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 25099 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::MachineMemOperand::getFlags(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTypeAction(), llvm::TargetLoweringBase::getTypeToTransformTo(), llvm::SelectionDAG::getUNDEF(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), llvm::MVT::is32BitVector(), llvm::MVT::is64BitVector(), isFreeToSplitVector(), llvm::MVT::isInteger(), llvm::StoreSDNode::isTruncatingStore(), llvm::EVT::isVector(), splitVectorStore(), llvm::ISD::TRUNCATE, llvm::TargetLoweringBase::TypeWidenVector, and llvm::X86ISD::VEXTRACT_STORE.

Referenced by llvm::X86TargetLowering::LowerOperation().

lowerToAddSubOrFMAddSub()

static SDValue lowerToAddSubOrFMAddSub ( const BuildVectorSDNode *  BV, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or X86ISD::FMSUBADD node.

Definition at line 8229 of file X86ISelLowering.cpp.

References llvm::Add, llvm::X86ISD::ADDSUB, DL, llvm::ISD::FADD, llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMSUBADD, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::MVT::is512BitVector(), isAddSubOrSubAdd(), and isFMAddSubOrFMSubAdd().

LowerToHorizontalOp()

static SDValue LowerToHorizontalOp ( const BuildVectorSDNode *  BV, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Lower BUILD_VECTOR to a horizontal add/sub operation if possible.

Definition at line 8396 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::count_if(), DL, ExpandHorizontalBinOp(), llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, getHopForBuildVector(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, llvm::MVT::is256BitVector(), isHopBuildVector(), isHorizontalBinOpPart(), llvm::SDValue::isUndef(), llvm::SDNode::isUndef(), llvm::SDNode::op_values(), and llvm::ISD::SUB.

LowerToTLSExecModel()

static SDValue LowerToTLSExecModel ( GlobalAddressSDNode *  GA, SelectionDAG &  DAG, const EVT  PtrVT, TLSModel::Model  model, bool  is64Bit, bool  isPIC  )

static

Definition at line 19099 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::X86AS::FS, llvm::PointerType::get(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getEntryNode(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachinePointerInfo::getGOT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SDNode::getValueType(), llvm::X86ISD::GlobalBaseReg, llvm::X86AS::GS, llvm::TLSModel::InitialExec, is64Bit(), llvm_unreachable, llvm::TLSModel::LocalExec, llvm::X86II::MO_GOTNTPOFF, llvm::X86II::MO_GOTTPOFF, llvm::X86II::MO_INDNTPOFF, llvm::X86II::MO_NTPOFF, llvm::X86II::MO_TPOFF, llvm::Offset, Ptr, llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.

LowerToTLSGeneralDynamicModel32()

static SDValue LowerToTLSGeneralDynamicModel32 ( GlobalAddressSDNode *  GA, SelectionDAG &  DAG, const EVT  PtrVT  )

static

Definition at line 19041 of file X86ISelLowering.cpp.

References GetTLSADDR(), and llvm::X86II::MO_TLSGD.

LowerToTLSGeneralDynamicModel64()

static SDValue LowerToTLSGeneralDynamicModel64 ( GlobalAddressSDNode *  GA, SelectionDAG &  DAG, const EVT  PtrVT  )

static

Definition at line 19049 of file X86ISelLowering.cpp.

References GetTLSADDR(), and llvm::X86II::MO_TLSGD.

LowerToTLSGeneralDynamicModelX32()

static SDValue LowerToTLSGeneralDynamicModelX32 ( GlobalAddressSDNode *  GA, SelectionDAG &  DAG, const EVT  PtrVT  )

static

Definition at line 19056 of file X86ISelLowering.cpp.

References GetTLSADDR(), and llvm::X86II::MO_TLSGD.

LowerToTLSLocalDynamicModel()

static SDValue LowerToTLSLocalDynamicModel ( GlobalAddressSDNode *  GA, SelectionDAG &  DAG, const EVT  PtrVT, bool  Is64Bit, bool  Is64BitLP64  )

static

Definition at line 19061 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::sampleprof::Base, llvm::GlobalAddressSDNode::getGlobal(), llvm::MachineFunction::getInfo(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), GetTLSADDR(), llvm::SDNode::getValueType(), llvm::X86MachineFunctionInfo::incNumLocalDynamicTLSAccesses(), llvm::X86II::MO_DTPOFF, llvm::X86II::MO_TLSLD, llvm::X86II::MO_TLSLDM, llvm::Offset, and llvm::X86ISD::Wrapper.

LowerTruncateVecI1()

static SDValue LowerTruncateVecI1 ( SDValue  Op, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 20933 of file X86ISelLowering.cpp.

References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::CONCAT_VECTORS, DL, extract128BitVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::Lo, llvm::ISD::SETGT, llvm::ISD::SETNE, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, and llvm::ISD::TRUNCATE.

LowerTruncateVecPack()

static SDValue LowerTruncateVecPack ( MVT  DstVT, SDValue  In, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

This function lowers a vector truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations.

Definition at line 20882 of file X86ISelLowering.cpp.

References DL, llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::isPowerOf2_32(), isUpperSubvectorUndef(), llvm::Lo, LowerTruncateVecPack(), llvm::ISD::TRUNCATE, truncateVectorWithPACKSS(), truncateVectorWithPACKUS(), and widenSubVector().

Referenced by LowerTruncateVecPack().

LowerTruncateVecPackWithSignBits()

static SDValue LowerTruncateVecPackWithSignBits ( MVT  DstVT, SDValue  In, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

This function lowers a vector truncation of 'extended sign-bits' or 'extended zero-bits' values.

vXi16/vXi32/vXi64 to vXi8/vXi16/vXi32 into X86ISD::PACKSS/PACKUS operations.

Definition at line 20848 of file X86ISelLowering.cpp.

References DL, llvm::MVT::getHalfNumVectorElementsVT(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), isUpperSubvectorUndef(), llvm::Lo, LowerTruncateVecPackWithSignBits(), matchTruncateWithPACK(), truncateVectorWithPACK(), and widenSubVector().

Referenced by LowerTruncateVecPackWithSignBits().

LowerUINT_TO_FP_i32()

static SDValue LowerUINT_TO_FP_i32 ( SDValue  Op, const SDLoc &  dl, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

32-bit unsigned integer to float expansion.

Definition at line 19871 of file X86ISelLowering.cpp.

References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getFPExtendOrRound(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), getShuffleVectorZeroOrUndef(), llvm::SelectionDAG::getStrictFPExtendOrRound(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::Or, llvm::ISD::OR, llvm::ISD::SCALAR_TO_VECTOR, and llvm::ISD::STRICT_FSUB.

LowerUINT_TO_FP_i64()

static SDValue LowerUINT_TO_FP_i64 ( SDValue  Op, const SDLoc &  dl, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

64-bit unsigned integer to double expansion.

Definition at line 19803 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::ISD::FSUB, llvm::ConstantVector::get(), llvm::ConstantDataVector::get(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetLoweringInfo(), getUnpackl(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE3(), llvm::APFloatBase::IEEEdouble(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, and shouldUseHorizontalOp().

lowerUINT_TO_FP_v2i32()

static SDValue lowerUINT_TO_FP_v2i32 ( SDValue  Op, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 19921 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTUI2P, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::X86Subtarget::hasAVX512(), llvm::Or, llvm::ISD::OR, llvm::X86ISD::STRICT_CVTUI2P, llvm::ISD::STRICT_FSUB, and llvm::ISD::ZERO_EXTEND.

Referenced by lowerUINT_TO_FP_vec().

lowerUINT_TO_FP_vec()

static SDValue lowerUINT_TO_FP_vec ( SDValue  Op, const SDLoc &  dl, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 20132 of file X86ISelLowering.cpp.

References llvm::SDValue::getSimpleValueType(), llvm_unreachable, lowerINT_TO_FP_vXi64(), lowerUINT_TO_FP_v2i32(), lowerUINT_TO_FP_vXi32(), and llvm::MVT::SimpleTy.

lowerUINT_TO_FP_vXi32()

static SDValue lowerUINT_TO_FP_vXi32 ( SDValue  Op, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 19974 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::X86ISD::BLENDI, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE41(), High, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEsingle(), llvm::ISD::INSERT_SUBVECTOR, llvm::Low, llvm::MachineMemOperand::MOLoad, llvm::Or, llvm::ISD::OR, llvm::ISD::SRL, llvm::ISD::STRICT_FADD, llvm::ISD::STRICT_FSUB, llvm::ISD::STRICT_UINT_TO_FP, llvm::ISD::UINT_TO_FP, llvm::X86ISD::VBROADCAST_LOAD, and llvm::ISD::ZERO_EXTEND.

Referenced by lowerUINT_TO_FP_vec().

lowerV16F32Shuffle()

static SDValue lowerV16F32Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 16-lane 32-bit floating point shuffles.

Definition at line 17036 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), lowerShuffleAsBlend(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithEXPAND(), lowerShuffleWithPERMV(), lowerShuffleWithSHUFPS(), lowerShuffleWithUNPCK(), llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::VPERMILPI, and llvm::X86ISD::VPERMILPV.

Referenced by lower512BitShuffle().

lowerV16I16Shuffle()

static SDValue lowerV16I16Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 16-lane 16-bit integer shuffles.

This routine is only called when we have AVX2 and thus a reasonable instruction set for v16i16 shuffling..

Definition at line 16559 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsSplitOrBlend(), lowerShuffleAsVTRUNC(), lowerShuffleAsZeroOrAnyExtend(), lowerShufflePairAsUNPCKAndPermute(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithPSHUFB(), lowerShuffleWithUNPCK(), lowerShuffleWithUNPCK256(), and lowerV8I16GeneralSingleInputShuffle().

Referenced by lower256BitShuffle().

lowerV16I32Shuffle()

static SDValue lowerV16I32Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 16-lane 32-bit integer shuffles.

Definition at line 17176 of file X86ISelLowering.cpp.

References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), is128BitLaneRepeatedShuffleMask(), isSingleSHUFPSMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsByteRotate(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsVALIGN(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithEXPAND(), lowerShuffleWithPERMV(), lowerShuffleWithSHUFPS(), lowerShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, and llvm::SmallVectorBase< Size_T >::size().

Referenced by lower512BitShuffle().

lowerV16I8Shuffle()

static SDValue lowerV16I8Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Generic lowering of v16i8 shuffles.

This is a hybrid strategy to lower v16i8 vectors. It first attempts to detect any complexity reducing interleaving. If that doesn't help, it uses UNPCK to spread the i8 elements across two i16-element vectors, and uses the existing lowering for v8i16 blends on each half, finally PACK-ing them back together.

Definition at line 14372 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::array_pod_sort(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), canLowerByDroppingElements(), llvm::copy_if(), llvm::count_if(), DL, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), I, lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBlendOfPSHUFBs(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsByteShiftMask(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsShift(), lowerShuffleAsVTRUNC(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithSSE4A(), lowerShuffleWithUNPCK(), lowerShuffleWithVPMOV(), llvm::Masked, llvm::none_of(), llvm::X86ISD::PACKUS, llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::unique(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VPPERM, and llvm::X86ISD::VSRLI.

Referenced by lower128BitShuffle().

lowerV2F64Shuffle()

static SDValue lowerV2F64Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 2-lane 64-bit floating point shuffles.

This is the basis function for the 2-lane 64-bit shuffles as we have full support for floating point shuffles but not integer shuffles. These instructions will incur a domain crossing penalty on some chips though so it is better to avoid lowering through this for integer vectors where possible.

Definition at line 13007 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), getScalarValueForVectorElement(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), isShuffleEquivalent(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsElementInsertion(), lowerShuffleOfExtractsAsVperm(), lowerShuffleWithUNPCK(), llvm::X86ISD::MOVSD, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::SHUFP, llvm::SM_SentinelUndef, and llvm::X86ISD::VPERMILPI.

Referenced by lower128BitShuffle().

lowerV2I64Shuffle()

static SDValue lowerV2I64Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 2-lane 64-bit integer shuffles.

Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by the integer unit to minimize domain crossing penalties. However, for blends it falls back to the floating point shuffle operation with appropriate bit casting.

Definition at line 13091 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsShift(), lowerShuffleAsVALIGN(), lowerShuffleOfExtractsAsVperm(), lowerShuffleWithUNPCK(), and llvm::X86ISD::PSHUFD.

Referenced by lower128BitShuffle().

lowerV2X128Shuffle()

static SDValue lowerV2X128Shuffle ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering 2-lane 128-bit shuffles.

Definition at line 15174 of file X86ISelLowering.cpp.

References assert(), canWidenShuffleElements(), DL, llvm::ISD::EXTRACT_SUBVECTOR, getBROADCAST_LOAD(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getStoreSize(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), isShuffleEquivalent(), lowerShuffleAsBlend(), llvm::X86::mayFoldLoad(), llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::SHUF128, llvm::X86ISD::SUBV_BROADCAST_LOAD, and llvm::X86ISD::VPERM2X128.

Referenced by lowerV4F64Shuffle(), and lowerV4I64Shuffle().

lowerV32I16Shuffle()

static SDValue lowerV32I16Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 32-lane 16-bit integer shuffles.

Definition at line 17273 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SDValue::getSimpleValueType(), is128BitLaneRepeatedShuffleMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsByteRotate(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsShift(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithPSHUFB(), lowerShuffleWithUNPCK(), and lowerV8I16GeneralSingleInputShuffle().

Referenced by lower512BitShuffle().

lowerV32I8Shuffle()

static SDValue lowerV32I8Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 32-lane 8-bit integer shuffles.

This routine is only called when we have AVX2 and thus a reasonable instruction set for v32i8 shuffling..

Definition at line 16682 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), is128BitLaneCrossingShuffleMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsSplitOrBlend(), lowerShuffleAsVTRUNC(), lowerShuffleAsVTRUNCAndUnpack(), lowerShuffleAsZeroOrAnyExtend(), lowerShufflePairAsUNPCKAndPermute(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithPSHUFB(), lowerShuffleWithUNPCK(), and lowerShuffleWithUNPCK256().

Referenced by lower256BitShuffle().

lowerV4F32Shuffle()

static SDValue lowerV4F32Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Lower 4-lane 32-bit floating point shuffles.

Uses instructions exclusively from the floating point unit to minimize domain crossing penalties, as these are sufficient to implement all v4f32 shuffles.

Definition at line 13283 of file X86ISelLowering.cpp.

References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), isShuffleEquivalent(), isSingleSHUFPSMask(), lowerShuffleAsBlend(), lowerShuffleAsBlendAndPermute(), lowerShuffleAsBroadcast(), lowerShuffleAsElementInsertion(), lowerShuffleAsInsertPS(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleOfExtractsAsVperm(), lowerShuffleWithSHUFPS(), lowerShuffleWithUNPCK(), llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::SHUFP, and llvm::X86ISD::VPERMILPI.

Referenced by lower128BitShuffle().

lowerV4F64Shuffle()

static SDValue lowerV4F64Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 4-lane 64-bit floating point shuffles.

Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2 isn't available.

Definition at line 16062 of file X86ISelLowering.cpp.

References llvm::all_of(), assert(), llvm::ISD::BUILD_VECTOR, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), is128BitLaneCrossingShuffleMask(), isShuffleEquivalent(), isShuffleMaskInputInPlace(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsLanePermuteAndSHUFP(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsSplitOrBlend(), lowerShuffleWithEXPAND(), lowerShuffleWithSHUFPD(), lowerShuffleWithUNPCK(), lowerV2X128Shuffle(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.

Referenced by lower256BitShuffle().

lowerV4I32Shuffle()

static SDValue lowerV4I32Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Lower 4-lane i32 vector shuffles.

We try to handle these with integer-domain shuffles where we can, but for blends we use the floating point domain blend instructions.

Definition at line 13387 of file X86ISelLowering.cpp.

References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), if(), isShuffleEquivalent(), isSingleSHUFPSMask(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsShift(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleOfExtractsAsVperm(), lowerShuffleWithUNPCK(), llvm::Masked, and llvm::X86ISD::PSHUFD.

Referenced by lower128BitShuffle().

lowerV4I64Shuffle()

static SDValue lowerV4I64Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 4-lane 64-bit integer shuffles.

This routine is only called when we have AVX2 and thus a reasonable instruction set for v4i64 shuffling..

Definition at line 16182 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), is128BitLaneRepeatedShuffleMask(), isShuffleMaskInputInPlace(), lowerShuffleAsBlend(), lowerShuffleAsBlendAndPermute(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsVALIGN(), lowerShuffleWithEXPAND(), lowerShuffleWithUNPCK(), lowerV2X128Shuffle(), llvm::narrowShuffleMaskElts(), llvm::X86ISD::PSHUFD, and llvm::X86ISD::VPERMI.

Referenced by lower256BitShuffle().

lowerV4X128Shuffle()

static SDValue lowerV4X128Shuffle ( const SDLoc &  DL, MVT  VT, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to lower a vector shuffle as a 128-bit shuffles.

Definition at line 16876 of file X86ISelLowering.cpp.

References assert(), canWidenShuffleElements(), llvm::SmallVectorImpl< T >::clear(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getZeroVector(), insert128BitVector(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is512BitVector(), isShuffleEquivalent(), isUndef(), llvm::narrowShuffleMaskElts(), OpIndex, llvm::X86ISD::SHUF128, and llvm::SmallVectorBase< Size_T >::size().

Referenced by lowerV8F64Shuffle(), and lowerV8I64Shuffle().

lowerV64I8Shuffle()

static SDValue lowerV64I8Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 64-lane 8-bit integer shuffles.

Definition at line 17344 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SDValue::getSimpleValueType(), is128BitLaneCrossingShuffleMask(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBlendOfPSHUFBs(), lowerShuffleAsByteRotate(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithPSHUFB(), lowerShuffleWithUNPCK(), llvm::Masked, and splitAndLowerShuffle().

Referenced by lower512BitShuffle().

lowerV8F16Shuffle()

static SDValue lowerV8F16Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Lower 8-lane 16-bit floating point shuffles.

Definition at line 14289 of file X86ISelLowering.cpp.

References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getVectorShuffle(), lowerShuffleAsBroadcast(), and lowerShuffleAsElementInsertion().

Referenced by lower128BitShuffle().

lowerV8F32Shuffle()

static SDValue lowerV8F32Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 8-lane 32-bit floating point shuffles.

Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2 isn't available.

Definition at line 16295 of file X86ISelLowering.cpp.

References assert(), computeInLaneShuffleMask(), DL, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), isUnpackWdShuffleMask(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsSplitOrBlend(), lowerShuffleAsZeroOrAnyExtend(), lowerShufflePairAsUNPCKAndPermute(), lowerShuffleWithEXPAND(), lowerShuffleWithSHUFPS(), lowerShuffleWithUNPCK(), llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::SmallVectorBase< Size_T >::size(), splitAndLowerShuffle(), llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, and llvm::X86ISD::VPERMV.

Referenced by lower256BitShuffle().

lowerV8F64Shuffle()

static SDValue lowerV8F64Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 8-lane 64-bit floating point shuffles.

Definition at line 16982 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), getV4X86ShuffleImm8ForMask(), is128BitLaneCrossingShuffleMask(), is256BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), lowerShuffleAsBlend(), lowerShuffleWithEXPAND(), lowerShuffleWithPERMV(), lowerShuffleWithSHUFPD(), lowerShuffleWithUNPCK(), lowerV4X128Shuffle(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.

Referenced by lower512BitShuffle().

lowerV8I16GeneralSingleInputShuffle()

static SDValue lowerV8I16GeneralSingleInputShuffle ( const SDLoc &  DL, MVT  VT, SDValue  V, MutableArrayRef< int >  Mask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.

The lowering strategy is to try to form pairs of input lanes which are targeted at the same half of the final vector, and then use a dword shuffle to place them onto the right half, and finally unpack the paired lanes into their final position.

The exact breakdown of how to form these dword pairs and align them on the correct sides is really tricky. See the comments within the function for more of the details.

This code also handles repeated 128-bit lanes of v8i16 shuffles, but each lane must shuffle the exact same way. In fact, you must pass a v8 Mask to this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16 vector, form the analogous 128-bit 8-element Mask.

Definition at line 13528 of file X86ISelLowering.cpp.

References llvm::array_pod_sort(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), llvm::copy_if(), llvm::count(), llvm::count_if(), llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::ArrayRef< T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::find(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::is_contained(), isNoopShuffleMask(), isSequentialOrUndefInRange(), isUndefOrEqual(), isUndefOrInRange(), llvm_unreachable, llvm::lower_bound(), lowerV8I16GeneralSingleInputShuffle(), llvm::M0(), llvm::M1(), llvm::Match, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), std::swap(), and llvm::unique().

Referenced by lowerV16I16Shuffle(), lowerV32I16Shuffle(), lowerV8I16GeneralSingleInputShuffle(), and lowerV8I16Shuffle().

lowerV8I16Shuffle()

static SDValue lowerV8I16Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Generic lowering of 8-lane i16 shuffles.

This handles both single-input shuffles and combined shuffle/blends with two inputs. The single input shuffles are immediately delegated to a dedicated lowering routine.

The blends are lowered in one of three fundamental ways. If there are few enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle of the input is significantly cheaper when lowered as an interleaving of the two inputs, try to interleave them. Otherwise, blend the low and high halves of the inputs separately (making them have relatively few inputs) and then concatenate them.

Definition at line 14085 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::any_of(), assert(), llvm::X86ISD::BLENDI, canLowerByDroppingElements(), concatSubVectors(), llvm::count_if(), DL, extract128BitVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getOpcode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBlendOfPSHUFBs(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsByteShiftMask(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsShift(), lowerShuffleAsVTRUNC(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithPACK(), lowerShuffleWithSSE4A(), lowerShuffleWithUNPCK(), lowerShuffleWithVPMOV(), lowerV8I16GeneralSingleInputShuffle(), llvm::Masked, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.

Referenced by lower128BitShuffle().

lowerV8I32Shuffle()

static SDValue lowerV8I32Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 8-lane 32-bit integer shuffles.

This routine is only called when we have AVX2 and thus a reasonable instruction set for v8i32 shuffling..

Definition at line 16417 of file X86ISelLowering.cpp.

References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), is128BitLaneRepeatedShuffleMask(), isSingleSHUFPSMask(), isUnpackWdShuffleMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsSplitOrBlend(), lowerShuffleAsVALIGN(), lowerShuffleAsZeroOrAnyExtend(), lowerShufflePairAsUNPCKAndPermute(), lowerShuffleWithEXPAND(), lowerShuffleWithSHUFPS(), lowerShuffleWithUNPCK(), lowerShuffleWithUNPCK256(), llvm::X86ISD::PSHUFD, llvm::SmallVectorBase< Size_T >::size(), and llvm::X86ISD::VPERMV.

Referenced by lower256BitShuffle().

lowerV8I64Shuffle()

static SDValue lowerV8I64Shuffle ( const SDLoc &  DL, ArrayRef< int >  Mask, const APInt &  Zeroable, SDValue  V1, SDValue  V2, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Handle lowering of 8-lane 64-bit integer shuffles.

Definition at line 17103 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), is128BitLaneRepeatedShuffleMask(), is256BitLaneRepeatedShuffleMask(), lowerShuffleAsBlend(), lowerShuffleAsByteRotate(), lowerShuffleAsShift(), lowerShuffleAsVALIGN(), lowerShuffleWithEXPAND(), lowerShuffleWithPERMV(), lowerShuffleWithUNPCK(), lowerV4X128Shuffle(), llvm::narrowShuffleMaskElts(), llvm::X86ISD::PSHUFD, and llvm::X86ISD::VPERMI.

Referenced by lower512BitShuffle().

LowerVACOPY()

static SDValue LowerVACOPY ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 25559 of file X86ISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::expandVACopy(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemcpy(), llvm::X86Subtarget::isCallingConvWin64(), and llvm::X86Subtarget::isTarget64BitLP64().

Referenced by llvm::X86TargetLowering::LowerOperation().

lowerVECTOR_COMPRESS()

static SDValue lowerVECTOR_COMPRESS ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 18001 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::EVT::getFixedSizeInBits(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::isUndef(), llvm::ISD::TRUNCATE, llvm::ISD::VECTOR_COMPRESS, and widenSubVector().

Referenced by llvm::X86TargetLowering::LowerOperation().

lowerVECTOR_SHUFFLE()

static SDValue lowerVECTOR_SHUFFLE ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Top-level lowering for x86 vector shuffles.

This handles decomposition, canonicalization, and lowering of all x86 vector shuffles. Most of the specific lowering strategies are encapsulated above in helper routines. The canonicalization attempts to widen shuffles to involve fewer lanes of wider elements, consolidate symmetric patterns s.t. only one of the two inputs needs to be tested, etc.

Definition at line 17858 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::any_of(), assert(), canCombineAsMaskOperation(), canonicalizeShuffleMaskWithCommute(), canonicalizeShuffleMaskWithHorizOp(), canWidenShuffleElements(), llvm::ShuffleVectorSDNode::commuteMask(), computeZeroableShuffleElements(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getCommutedVectorShuffle(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::ShuffleVectorSDNode::getMask(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::is_contained(), llvm::APInt::isAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm_unreachable, lower128BitShuffle(), lower1BitShuffle(), lower256BitShuffle(), lower512BitShuffle(), lowerShuffleAsBroadcast(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelZero, and std::swap().

LowerVectorAllEqual()

static SDValue LowerVectorAllEqual ( const SDLoc &  DL, SDValue  LHS, SDValue  RHS, ISD::CondCode  CC, const APInt &  OriginalMask, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, X86::CondCode &  X86CC  )

static

Definition at line 22605 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), CC, llvm::EVT::changeVectorElementType(), llvm::MVT::changeVectorElementType(), llvm::X86ISD::CMP, llvm::SelectionDAG::computeKnownBits(), llvm::X86::COND_E, llvm::X86::COND_NE, DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::KnownBits::getConstant(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, llvm::EVT::is512BitVector(), llvm::KnownBits::isConstant(), llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::KnownBits::isZero(), llvm::X86ISD::KORTEST, LHS, llvm::Lo, llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PTEST, RHS, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::SelectionDAG::SplitScalar(), llvm::SelectionDAG::SplitVector(), llvm::X86Subtarget::useAVX512Regs(), and llvm::ISD::XOR.

Referenced by combinePredicateReduction(), and MatchVectorAllEqualTest().

LowerVectorCTLZ()

static SDValue LowerVectorCTLZ ( SDValue  Op, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 28595 of file X86ISelLowering.cpp.

References assert(), llvm::X86Subtarget::canExtendTo512DQ(), DL, llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerVectorCTLZ_AVX512CDI(), LowerVectorCTLZInRegLUT(), and splitVectorIntUnary().

Referenced by LowerCTLZ().

LowerVectorCTLZ_AVX512CDI()

static SDValue LowerVectorCTLZ_AVX512CDI ( SDValue  Op, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Lower a vector CTLZ using native supported vector CTLZ instruction.

Definition at line 28483 of file X86ISelLowering.cpp.

References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::CTLZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), splitVectorIntUnary(), llvm::ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by LowerVectorCTLZ().

LowerVectorCTLZInRegLUT()

static SDValue LowerVectorCTLZInRegLUT ( SDValue  Op, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 28513 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::Hi, llvm::Lo, LUT, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SETEQ, llvm::ISD::SIGN_EXTEND, and llvm::ISD::SRL.

Referenced by LowerVectorCTLZ().

LowerVectorCTPOP()

static SDValue LowerVectorCTPOP ( SDValue  Op, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 32054 of file X86ISelLowering.cpp.

References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::CTPOP, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerHorizontalByteSum(), LowerVectorCTPOPInRegLUT(), splitVectorIntUnary(), llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by LowerCTPOP().

LowerVectorCTPOPInRegLUT()

static SDValue LowerVectorCTPOPInRegLUT ( SDValue  Op, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 32007 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, assert(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), LUT, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::SRL.

Referenced by LowerVectorCTPOP().

lowerVSELECTtoVectorShuffle()

static SDValue lowerVSELECTtoVectorShuffle ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Try to lower a VSELECT instruction to a vector shuffle.

Definition at line 18062 of file X86ISelLowering.cpp.

References Cond, createShuffleMaskFromVSELECT(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::isBuildVectorOfConstantSDNodes(), LHS, and RHS.

LowerVSETCC()

static SDValue LowerVSETCC ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 23560 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::BitWidth, CC, llvm::MVT::changeVectorElementType(), llvm::EVT::changeVectorElementTypeToInteger(), cheapX86FSETCC_SSE(), llvm::X86ISD::CMPM, llvm::X86ISD::CMPP, llvm::SelectionDAG::ComputeNumSignBits(), Cond, EQ, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getAllOnesConstant(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getFixedSizeInBits(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE42(), incDecVectorConstant(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), isConstantPowerOf2(), llvm::ISD::isConstantSplatVector(), llvm::isConstOrConstSplat(), llvm::MVT::isFloatingPoint(), llvm::APInt::isMaxSignedValue(), llvm::APInt::isMinSignedValue(), llvm::TargetLoweringBase::isOperationLegal(), llvm::APInt::isPowerOf2(), isSoftF16(), llvm::ISD::isTrueWhenEqual(), llvm::ISD::isUnsignedIntSetCC(), llvm::APInt::isZero(), llvm_unreachable, llvm::APInt::logBase2(), LowerIntVSETCC_AVX512(), LowerVSETCCWithSUBUS(), llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), llvm::ISD::SETEQ, llvm::SDNode::setFlags(), llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::SelectionDAG::SignBitIsZero(), splitVSETCC(), llvm::ISD::SRA, llvm::X86ISD::STRICT_CMPM, llvm::X86ISD::STRICT_CMPP, llvm::ISD::STRICT_FP_EXTEND, llvm::ISD::STRICT_FSETCC, llvm::ISD::STRICT_FSETCCS, std::swap(), llvm::ISD::TokenFactor, translateX86FSETCC(), llvm::ISD::TRUNCATE, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::X86Subtarget::useAVX512Regs(), llvm::X86ISD::VPCOM, llvm::X86ISD::VPCOMU, and llvm::ISD::XOR.

LowerVSETCCWithSUBUS()

static SDValue LowerVSETCCWithSUBUS ( SDValue  Op0, SDValue  Op1, MVT  VT, ISD::CondCode  Cond, const SDLoc &  dl, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

As another special case, use PSUBUS[BW] when it's profitable.

E.g. for Op0 u<= Op1: t = psubus Op0, Op1 pcmpeq t, <0..0>

Definition at line 23505 of file X86ISelLowering.cpp.

References Cond, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE2(), incDecVectorConstant(), llvm::X86ISD::PCMPEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, std::swap(), and llvm::ISD::USUBSAT.

Referenced by LowerVSETCC().

LowervXi8MulWithUNPCK()

static SDValue LowervXi8MulWithUNPCK ( SDValue  A, SDValue  B, const SDLoc &  dl, MVT  VT, bool  IsSigned, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, SDValue *  Low = nullptr  )

static

Definition at line 29209 of file X86ISelLowering.cpp.

References A, B, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPack(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::Low, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::SHL.

Referenced by LowerMULH(), and LowerMULO().

lowerX86CmpEqZeroToCtlzSrl()

static SDValue lowerX86CmpEqZeroToCtlzSrl ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 51312 of file X86ISelLowering.cpp.

References llvm::ISD::CTLZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::Log2_32(), and llvm::ISD::SRL.

Referenced by combineOrCmpEqZeroToCtlzSrl().

lowerX86FPLogicOp()

static SDValue lowerX86FPLogicOp ( SDNode *  N, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 54007 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::X86ISD::FAND, llvm::X86ISD::FANDN, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::isVector(), llvm_unreachable, N, llvm::ISD::OR, and llvm::ISD::XOR.

Referenced by combineFAnd(), combineFAndn(), and combineFOr().

LowerXALUO()

static SDValue LowerXALUO ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 24348 of file X86ISelLowering.cpp.

References assert(), Cond, DL, llvm::SelectionDAG::getNode(), getSETCC(), getX86XALUOOp(), and llvm::ISD::MERGE_VALUES.

LowerZERO_EXTEND()

static SDValue LowerZERO_EXTEND ( SDValue  Op, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 20598 of file X86ISelLowering.cpp.

References assert(), DL, llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX(), LowerAVXExtend(), and LowerZERO_EXTEND_Mask().

Referenced by llvm::X86TargetLowering::LowerOperation().

LowerZERO_EXTEND_Mask()

static SDValue LowerZERO_EXTEND_Mask ( SDValue  Op, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 20541 of file X86ISelLowering.cpp.

References assert(), llvm::X86Subtarget::canExtendTo512DQ(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is512BitVector(), llvm::ISD::SIGN_EXTEND, SplitAndExtendv16i1(), llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by LowerZERO_EXTEND().

MarkEHGuard()

static SDValue MarkEHGuard ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 27192 of file X86ISelLowering.cpp.

References llvm::WinEHFuncInfo::EHGuardFrameIndex, llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getWinEHFuncInfo(), and llvm::report_fatal_error().

Referenced by LowerINTRINSIC_W_CHAIN().

MarkEHRegistrationNode()

static SDValue MarkEHRegistrationNode ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 27174 of file X86ISelLowering.cpp.

References llvm::WinEHFuncInfo::EHRegNodeFrameIndex, llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getWinEHFuncInfo(), and llvm::report_fatal_error().

Referenced by LowerINTRINSIC_W_CHAIN().

match1BitShuffleAsKSHIFT()

static int match1BitShuffleAsKSHIFT ( unsigned &  Opcode, ArrayRef< int >  Mask, int  MaskOffset, const APInt &  Zeroable  )

static

Definition at line 17557 of file X86ISelLowering.cpp.

References isSequentialOrUndefInRange(), llvm::X86ISD::KSHIFTL, llvm::X86ISD::KSHIFTR, llvm::Left, llvm::Low, and Size.

Referenced by lower1BitShuffle().

matchAsm()

static bool matchAsm ( StringRef  S, ArrayRef< const char * >  Pieces  )

static

Definition at line 59512 of file X86ISelLowering.cpp.

References llvm::StringRef::empty(), llvm::StringRef::find_first_not_of(), llvm::StringRef::starts_with(), and llvm::StringRef::substr().

Referenced by llvm::X86TargetLowering::ExpandInlineAsm().

matchBinaryPermuteShuffle()

static bool matchBinaryPermuteShuffle ( MVT  MaskVT, ArrayRef< int >  Mask, const APInt &  Zeroable, bool  AllowFloatDomain, bool  AllowIntDomain, SDValue &  V1, SDValue &  V2, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, unsigned &  Shuffle, MVT &  ShuffleVT, unsigned &  PermuteImm  )

static

Definition at line 39193 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::BLENDI, DL, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isAnyZero(), isRepeatedTargetShuffleMask(), isUndefInRange(), isUndefOrInRange(), isUndefOrZeroInRange(), llvm::Lo, llvm::M0(), llvm::M1(), matchShuffleAsBlend(), matchShuffleAsByteRotate(), matchShuffleAsElementRotate(), matchShuffleAsInsertPS(), matchShuffleWithSHUFPD(), llvm::Offset, llvm::X86ISD::PALIGNR, S1, llvm::X86ISD::SHUFP, llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, and llvm::X86ISD::VALIGN.

Referenced by combineX86ShuffleChain().

matchBinaryShuffle()

static bool matchBinaryShuffle ( MVT  MaskVT, ArrayRef< int >  Mask, bool  AllowFloatDomain, bool  AllowIntDomain, SDValue &  V1, SDValue &  V2, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, unsigned &  Shuffle, MVT &  SrcVT, MVT &  DstVT, bool  IsUnary  )

static

Definition at line 38997 of file X86ISelLowering.cpp.

References llvm::MVT::changeTypeToInteger(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::APInt::getOneBitSet(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::KnownBits::isAllOnes(), isTargetShuffleEquivalent(), llvm::KnownBits::isZero(), llvm_unreachable, llvm::SelectionDAG::MaskedVectorIsZero(), matchShuffleWithPACK(), matchShuffleWithUNPCK(), llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSS, llvm::KnownBits::One, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::APInt::setBit(), llvm::APInt::setBits(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, std::swap(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::KnownBits::Zero.

Referenced by combineX86ShuffleChain().

matchLogicBlend()

static bool matchLogicBlend ( SDNode *  N, SDValue &  X, SDValue &  Y, SDValue &  Mask  )

static

Definition at line 51219 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), N, llvm::ISD::OR, std::swap(), X, and Y.

Referenced by combineLogicBlendIntoPBLENDV().

matchPMADDWD()

static SDValue matchPMADDWD ( SelectionDAG &  DAG, SDNode *  N, const SDLoc &  DL, EVT  VT, const X86Subtarget &  Subtarget  )

static

Definition at line 56524 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::ISD::BUILD_VECTOR, canReduceVMulWidth(), DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasSSE2(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_ExtractElt(), llvm::PatternMatch::m_Value(), llvm::Mul, llvm::ISD::MUL, N, SplitOpsAndApply(), std::swap(), llvm::ISD::TRUNCATE, and llvm::X86ISD::VPMADDWD.

Referenced by combineAdd().

matchPMADDWD_2()

static SDValue matchPMADDWD_2 ( SelectionDAG &  DAG, SDNode *  N, const SDLoc &  DL, EVT  VT, const X86Subtarget &  Subtarget  )

static

Definition at line 56634 of file X86ISelLowering.cpp.

References assert(), llvm::EVT::bitsLT(), llvm::ISD::BUILD_VECTOR, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasSSE2(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_ExtractElt(), llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_SExt(), llvm::PatternMatch::m_Value(), N, SplitOpsAndApply(), std::swap(), and llvm::X86ISD::VPMADDWD.

Referenced by combineAdd().

matchScalarReduction()

static bool matchScalarReduction ( SDValue  Op, ISD::NodeType  BinOp, SmallVectorImpl< SDValue > &  SrcOps, SmallVectorImpl< APInt > *  SrcMask = nullptr  )

static

Helper for matching BINOP(EXTRACTELT(X,0),BINOP(EXTRACTELT(X,1),...)) style scalarized (associative) reduction patterns.

Partial reductions are supported when the pointer SrcMask is non-null. TODO - move this to SelectionDAG?

Definition at line 22536 of file X86ISelLowering.cpp.

References assert(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::empty(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), I, Idx, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::insert(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by combineAnd(), combineOr(), and MatchVectorAllEqualTest().

matchShuffleAsBitRotate()

static int matchShuffleAsBitRotate ( MVT &  RotateVT, int  EltSizeInBits, const X86Subtarget &  Subtarget, ArrayRef< int >  Mask  )

static

Definition at line 11477 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getIntegerVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::ShuffleVectorInst::isBitRotateMask(), and isNoopShuffleMask().

matchShuffleAsBlend()

static bool matchShuffleAsBlend ( MVT  VT, SDValue  V1, SDValue  V2, MutableArrayRef< int >  Mask, const APInt &  Zeroable, bool &  ForceV1Zero, bool &  ForceV2Zero, uint64_t &  BlendMask  )

static

Definition at line 10761 of file X86ISelLowering.cpp.

References assert(), llvm::SDValue::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::is256BitVector(), llvm::ISD::isBuildVectorAllZeros(), IsElementEquivalent(), llvm::SDValue::isUndef(), and llvm::SM_SentinelUndef.

Referenced by lowerShuffleAsBlend(), and matchBinaryPermuteShuffle().

matchShuffleAsByteRotate()

static int matchShuffleAsByteRotate ( MVT  VT, SDValue &  V1, SDValue &  V2, ArrayRef< int >  Mask  )

static

Try to lower a vector shuffle as a byte rotation.

SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary byte-rotation of the concatenation of two vectors; pre-SSSE3 can use a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will try to generically lower a vector shuffle through such an pattern. It does not check for the profitability of lowering either as PALIGNR or PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form. This matches shuffle vectors that look like:

v8i16 [11, 12, 13, 14, 15, 0, 1, 2]

Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.

Definition at line 11626 of file X86ISelLowering.cpp.

References is128BitLaneRepeatedShuffleMask(), isAnyZero(), matchShuffleAsElementRotate(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by lowerShuffleAsByteRotate(), and matchBinaryPermuteShuffle().

matchShuffleAsElementRotate()

static int matchShuffleAsElementRotate ( SDValue &  V1, SDValue &  V2, ArrayRef< int >  Mask  )

static

Try to match a vector shuffle as an element rotation.

This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.

Definition at line 11541 of file X86ISelLowering.cpp.

References assert(), llvm::Hi, llvm::Lo, and llvm::SM_SentinelUndef.

Referenced by lowerShuffleAsVALIGN(), matchBinaryPermuteShuffle(), and matchShuffleAsByteRotate().

matchShuffleAsEXTRQ()

static bool matchShuffleAsEXTRQ ( MVT  VT, SDValue &  V1, SDValue &  V2, ArrayRef< int >  Mask, uint64_t &  BitLen, uint64_t &  BitIdx, const APInt &  Zeroable  )

static

Definition at line 11944 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), Idx, llvm::APInt::isAllOnes(), isUndefUpperHalf(), Size, and llvm::SM_SentinelUndef.

Referenced by combineX86ShuffleChain(), and lowerShuffleWithSSE4A().

matchShuffleAsInsertPS()

static bool matchShuffleAsInsertPS ( SDValue &  V1, SDValue &  V2, unsigned &  InsertPSMask, const APInt &  Zeroable, ArrayRef< int >  Mask, SelectionDAG &  DAG  )

static

Definition at line 12896 of file X86ISelLowering.cpp.

References assert(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), and llvm::MVT::is128BitVector().

Referenced by combineX86ShuffleChain(), lowerShuffleAsInsertPS(), and matchBinaryPermuteShuffle().

matchShuffleAsINSERTQ()

static bool matchShuffleAsINSERTQ ( MVT  VT, SDValue &  V1, SDValue &  V2, ArrayRef< int >  Mask, uint64_t &  BitLen, uint64_t &  BitIdx  )

static

Definition at line 12000 of file X86ISelLowering.cpp.

References assert(), llvm::sampleprof::Base, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::Hi, Idx, isSequentialOrUndefInRange(), isUndefInRange(), isUndefUpperHalf(), and Size.

Referenced by combineX86ShuffleChain(), and lowerShuffleWithSSE4A().

matchShuffleAsShift()

static int matchShuffleAsShift ( MVT &  ShiftVT, unsigned &  Opcode, unsigned  ScalarSizeInBits, ArrayRef< int >  Mask, int  MaskOffset, const APInt &  Zeroable, const X86Subtarget &  Subtarget  )

static

Try to lower a vector shuffle as a bit shift (shifts in zeros).

Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function matches elements from one of the input vectors shuffled to the left or right with zeroable elements 'shifted in'. It handles both the strictly bit-wise element shifts and the byte shift across an entire 128-bit double quad word lane.

PSHL : (little-endian) left bit shift. [ zz, 0, zz, 2 ] [ -1, 4, zz, -1 ] PSRL : (little-endian) right bit shift. [ 1, zz, 3, zz] [ -1, -1, 7, zz] PSLLDQ : (little-endian) left byte shift [ zz, 0, 1, 2, 3, 4, 5, 6] [ zz, zz, -1, -1, 2, 3, 4, -1] [ zz, zz, zz, zz, zz, zz, -1, 1] PSRLDQ : (little-endian) right byte shift [ 5, 6, 7, zz, zz, zz, zz, zz] [ -1, 5, 6, 7, zz, zz, zz, zz] [ 1, 2, -1, -1, -1, -1, zz, zz]

Definition at line 11843 of file X86ISelLowering.cpp.

References llvm::MVT::getIntegerVT(), llvm::MVT::getVectorVT(), isSequentialOrUndefInRange(), llvm::Left, llvm::Low, Size, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VSRLI.

Referenced by lowerShuffleAsShift(), and matchUnaryPermuteShuffle().

matchShuffleAsVTRUNC()

static bool matchShuffleAsVTRUNC ( MVT &  SrcVT, MVT &  DstVT, MVT  VT, ArrayRef< int >  Mask, const APInt &  Zeroable, const X86Subtarget &  Subtarget  )

static

Definition at line 10259 of file X86ISelLowering.cpp.

References llvm::APInt::extractBits(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorVT(), llvm::MVT::is512BitVector(), llvm::APInt::isAllOnes(), and isSequentialOrUndefInRange().

Referenced by combineX86ShuffleChain().

matchShuffleWithPACK()

static bool matchShuffleWithPACK ( MVT  VT, MVT &  SrcVT, SDValue &  V1, SDValue &  V2, unsigned &  PackOpcode, ArrayRef< int >  TargetMask, const SelectionDAG &  DAG, const X86Subtarget &  Subtarget, unsigned  MaxStages = 1  )

static

Definition at line 10562 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::ComputeNumSignBits(), createPackShuffleMask(), llvm::APInt::getHighBitsSet(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::isAllOnesOrAllOnesSplat(), llvm::isNullOrNullSplat(), isTargetShuffleEquivalent(), llvm::SDValue::isUndef(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, and llvm::peekThroughBitcasts().

Referenced by lowerShuffleWithPACK(), and matchBinaryShuffle().

matchShuffleWithSHUFPD()

static bool matchShuffleWithSHUFPD ( MVT  VT, SDValue &  V1, SDValue &  V2, bool &  ForceV1Zero, bool &  ForceV2Zero, unsigned &  ShuffleImm, ArrayRef< int >  Mask, const APInt &  Zeroable  )

static

Definition at line 15868 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getScalarSizeInBits(), getSHUFPDImm(), llvm::MVT::getVectorNumElements(), isUndefOrZeroOrInRange(), llvm::SM_SentinelUndef, and std::swap().

Referenced by lowerShuffleWithSHUFPD(), and matchBinaryPermuteShuffle().

matchShuffleWithUNPCK()

static bool matchShuffleWithUNPCK ( MVT  VT, SDValue &  V1, SDValue &  V2, unsigned &  UnpackOpcode, bool  IsUnary, ArrayRef< int >  TargetMask, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 10117 of file X86ISelLowering.cpp.

References assert(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::createUnpackShuffleMask(), DL, llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), isSequentialOrUndefOrZeroInRange(), isTargetShuffleEquivalent(), isUndefOrZero(), llvm::M1(), llvm::SM_SentinelUndef, std::swap(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by matchBinaryShuffle().

matchTruncateWithPACK()

static SDValue matchTruncateWithPACK ( unsigned &  PackOpcode, EVT  DstVT, SDValue  In, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Helper to determine if In truncated to DstVT has the necessary signbits / leading zero bits to be truncated with PACKSS / PACKUS, possibly by converting a SRL node to SRA for sign extension.

Definition at line 20758 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getValidShiftAmount(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), isFreeToSplitVector(), llvm::Log2_32(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::ISD::SRA, and llvm::ISD::SRL.

Referenced by LowerTruncateVecPackWithSignBits(), and llvm::X86TargetLowering::ReplaceNodeResults().

matchUnaryPermuteShuffle()

static bool matchUnaryPermuteShuffle ( MVT  MaskVT, ArrayRef< int >  Mask, const APInt &  Zeroable, bool  AllowFloatDomain, bool  AllowIntDomain, const SelectionDAG &  DAG, const X86Subtarget &  Subtarget, unsigned &  Shuffle, MVT &  ShuffleVT, unsigned &  PermuteImm  )

static

Definition at line 38848 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorTemplateCommon< T, typename >::data(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), getV4X86ShuffleImm(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is128BitVector(), is256BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isAnyZero(), isSequentialOrUndefInRange(), isUndefOrInRange(), matchShuffleAsBitRotate(), matchShuffleAsShift(), llvm::narrowShuffleMaskElts(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SM_SentinelUndef, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VROTLI, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.

Referenced by combineX86ShuffleChain().

matchUnaryShuffle()

static bool matchUnaryShuffle ( MVT  MaskVT, ArrayRef< int >  Mask, bool  AllowFloatDomain, bool  AllowIntDomain, SDValue  V1, const SelectionDAG &  DAG, const X86Subtarget &  Subtarget, unsigned &  Shuffle, MVT &  SrcVT, MVT &  DstVT  )

static

Definition at line 38694 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::MVT::changeVectorElementType(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::MVT::getIntegerVT(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isInteger(), isTargetShuffleEquivalent(), isUndefInRange(), isUndefOrEqual(), isUndefOrEqualInRange(), isUndefOrZero(), isUndefOrZeroInRange(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SIGN_EXTEND, llvm::X86ISD::VZEXT_MOVL, and llvm::ISD::ZERO_EXTEND.

Referenced by combineX86ShuffleChain().

MatchVectorAllEqualTest()

static SDValue MatchVectorAllEqualTest ( SDValue  LHS, SDValue  RHS, ISD::CondCode  CC, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, X86::CondCode &  X86CC  )

static

Definition at line 22753 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), CC, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::APInt::getZero(), llvm::X86Subtarget::hasSSE2(), llvm::isAllOnesConstant(), llvm::isNullConstant(), LHS, LowerVectorAllEqual(), llvm::Match, llvm::SelectionDAG::matchBinOpReduction(), matchScalarReduction(), llvm::ISD::OR, llvm::peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::SmallVectorBase< Size_T >::size(), and llvm::ISD::TRUNCATE.

Referenced by combineSetCC().

materializeVectorConstant()

static SDValue materializeVectorConstant ( SDValue  Op, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Create a vector constant without a load.

SSE/AVX provide the bare minimum functionality to do this, so it's all zeros, all ones, or some derivation that is cheap to calculate.

Definition at line 8597 of file X86ISelLowering.cpp.

References DL, getOnesVector(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), and llvm::ISD::isBuildVectorAllZeros().

narrowExtractedVectorSelect()

static SDValue narrowExtractedVectorSelect ( SDNode *  Ext, const SDLoc &  DL, SelectionDAG &  DAG  )

static

If we are extracting a subvector of a vector select and the select condition is composed of concatenated vectors, try to narrow the select width.

This is a common pattern for AVX1 integer code because 256-bit selects may be legal, but there is almost no integer math/logic available for 256-bit. This function should only be called with legal types (otherwise, the calls to get simple value types will assert).

Definition at line 58089 of file X86ISelLowering.cpp.

References assert(), DL, extract128BitVector(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isFreeToSplitVector(), llvm_unreachable, and llvm::ISD::VSELECT.

Referenced by combineEXTRACT_SUBVECTOR().

narrowLoadToVZLoad()

static SDValue narrowLoadToVZLoad ( LoadSDNode *  LN, MVT  MemVT, MVT  VT, SelectionDAG &  DAG  )

static

Definition at line 38678 of file X86ISelLowering.cpp.

References llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::SelectionDAG::getVTList(), llvm::MemSDNode::isSimple(), and llvm::X86ISD::VZEXT_LOAD.

Referenced by combineCVTP2I_CVTTP2I(), combineCVTPH2PS(), combineTargetShuffle(), and combineX86INT_TO_FP().

narrowShuffle()

static SDValue narrowShuffle ( ShuffleVectorSDNode *  Shuf, SelectionDAG &  DAG  )

static

If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the low half of each source vector and does not set any high half elements in the destination vector, narrow the shuffle to half its original size.

Definition at line 42665 of file X86ISelLowering.cpp.

References getHalfShuffleMask(), llvm::ShuffleVectorSDNode::getMask(), llvm::SDNode::getOperand(), getShuffleHalfVectors(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), and isUndefUpperHalf().

Referenced by combineShuffle().

narrowVectorSelect()

static SDValue narrowVectorSelect ( SDNode *  N, SelectionDAG &  DAG, const SDLoc &  DL, const X86Subtarget &  Subtarget  )

static

If both arms of a vector select are concatenated vectors, split the select, and concatenate the result to eliminate a wide (256-bit) vector instruction: vselect Cond, (concat T0, T1), (concat F0, F1) --> concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1)

Definition at line 46581 of file X86ISelLowering.cpp.

References llvm::X86ISD::BLENDV, Cond, DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::is256BitVector(), isFreeToSplitVector(), N, SplitOpsAndApply(), and llvm::ISD::VSELECT.

Referenced by combineSelect(), and llvm::InstCombinerImpl::visitShuffleVectorInst().

needCarryOrOverflowFlag()

static bool needCarryOrOverflowFlag ( SDValue  Flags )

static

Definition at line 56187 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::BRCOND, CC, llvm::X86ISD::CMOV, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NO, llvm::X86::COND_O, llvm::X86ISD::SETCC, and llvm::X86ISD::SETCC_CARRY.

Referenced by combineCMP().

negateFMAOpcode()

static unsigned negateFMAOpcode ( unsigned  Opcode, bool  NegMul, bool  NegAcc, bool  NegRes  )

static

Definition at line 53836 of file X86ISelLowering.cpp.

References llvm::ISD::FMA, llvm::X86ISD::FMADD_RND, llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMADDSUB_RND, llvm::X86ISD::FMSUB, llvm::X86ISD::FMSUB_RND, llvm::X86ISD::FMSUBADD, llvm::X86ISD::FMSUBADD_RND, llvm::X86ISD::FNMADD, llvm::X86ISD::FNMADD_RND, llvm::X86ISD::FNMSUB, llvm::X86ISD::FNMSUB_RND, llvm_unreachable, llvm::ISD::STRICT_FMA, llvm::X86ISD::STRICT_FMSUB, llvm::X86ISD::STRICT_FNMADD, and llvm::X86ISD::STRICT_FNMSUB.

onlyZeroFlagUsed()

static bool onlyZeroFlagUsed ( SDValue  Flags )

static

Definition at line 56222 of file X86ISelLowering.cpp.

References assert(), llvm::X86ISD::BRCOND, CC, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86ISD::SETCC, and llvm::X86ISD::SETCC_CARRY.

Referenced by combineCMP().

parseConstraintCode()

static X86::CondCode parseConstraintCode ( llvm::StringRef  Constraint )

static

Definition at line 59622 of file X86ISelLowering.cpp.

References llvm::StringSwitch< T, R >::Case(), Cond, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_INVALID, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm::X86::COND_NO, llvm::X86::COND_NP, llvm::X86::COND_NS, llvm::X86::COND_O, llvm::X86::COND_P, llvm::X86::COND_S, and llvm::StringSwitch< T, R >::Default().

promoteExtBeforeAdd()

static SDValue promoteExtBeforeAdd ( SDNode *  Ext, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

sext(add_nsw(x, C)) --> add(sext(x), C_sext) zext(add_nuw(x, C)) --> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode.

This can eliminate extend, add, and shift instructions.

Definition at line 54840 of file X86ISelLowering.cpp.

References llvm::Add, llvm::ISD::ADD, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::willNotOverflowAdd(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineSext(), and combineZext().

PromoteMaskArithmetic() [1/2]

static SDValue PromoteMaskArithmetic ( SDValue  N, const SDLoc &  DL, EVT  VT, SelectionDAG &  DAG, unsigned  Depth  )

static

Definition at line 50137 of file X86ISelLowering.cpp.

References llvm::Depth, DL, llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::ISD::isBitwiseLogicOp(), llvm::TargetLoweringBase::isOperationLegalOrPromote(), llvm::SelectionDAG::MaxRecursionDepth, N, PromoteMaskArithmetic(), llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineSext(), combineSignExtendInReg(), combineZext(), and PromoteMaskArithmetic().

PromoteMaskArithmetic() [2/2]

static SDValue PromoteMaskArithmetic ( SDValue  N, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Definition at line 50191 of file X86ISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::EVT::isVector(), llvm_unreachable, N, PromoteMaskArithmetic(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, and llvm::ISD::ZERO_EXTEND.

promoteXINT_TO_FP()

static SDValue promoteXINT_TO_FP ( SDValue  Op, const SDLoc &  dl, SelectionDAG &  DAG  )

static

Definition at line 19614 of file X86ISelLowering.cpp.

References llvm::MVT::changeVectorElementType(), llvm::ISD::FP_ROUND, llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::isVector(), and llvm::ISD::STRICT_FP_ROUND.

pushAddIntoCmovOfConsts()

static SDValue pushAddIntoCmovOfConsts ( SDNode *  N, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

CMOV of constants requires materializing constant operands in registers.

Try to fold those constants into an 'add' instruction to reduce instruction count. We do this with CMOV rather the generic 'select' because there are earlier folds that may be used to turn select-of-constants into logic hacks.

Definition at line 56793 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::all_of(), llvm::X86ISD::CMOV, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), llvm::X86::mayFoldLoad(), N, std::swap(), X, and Y.

Referenced by combineAdd().

rebuildGatherScatter()

static SDValue rebuildGatherScatter ( MaskedGatherScatterSDNode *  GorS, SDValue  Index, SDValue  Base, SDValue  Scale, SelectionDAG &  DAG  )

static

Definition at line 55745 of file X86ISelLowering.cpp.

References llvm::sampleprof::Base, DL, llvm::SelectionDAG::getMaskedGather(), and llvm::SelectionDAG::getMaskedScatter().

Referenced by combineGatherScatter().

recoverFramePointer()

static SDValue recoverFramePointer ( SelectionDAG &  DAG, const Function *  Fn, SDValue  EntryEBP  )

static

When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP.

Here's the math: RegNodeBase = EntryEBP - RegNodeSize ParentFP = RegNodeBase - ParentFrameOffset Subtracting RegNodeSize takes us to the offset of the registration node, and subtracting the offset (negative on x86) takes us back to the parent FP.

Definition at line 25863 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::GlobalValue::dropLLVMManglingEscape(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMCSymbol(), llvm::Value::getName(), llvm::SelectionDAG::getNode(), llvm::MCContext::getOrCreateParentFrameOffsetSymbol(), llvm::TargetLoweringBase::getPointerTy(), getSEHRegistrationNodeSize(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasPersonalityFn(), llvm::ISD::LOCAL_RECOVER, and llvm::ISD::SUB.

reduceMaskedLoadToScalarLoad()

static SDValue reduceMaskedLoadToScalarLoad ( MaskedLoadSDNode *  ML, SelectionDAG &  DAG, TargetLowering::DAGCombinerInfo &  DCI, const X86Subtarget &  Subtarget  )

static

If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert.

Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.

Definition at line 52396 of file X86ISelLowering.cpp.

References Addr, assert(), llvm::EVT::changeVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_VECTOR_ELT, ML, and llvm::Offset.

Referenced by combineMaskedLoad().

reduceMaskedStoreToScalarStore()

static SDValue reduceMaskedStoreToScalarStore ( MaskedStoreSDNode *  MS, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store.

Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.

Definition at line 52532 of file X86ISelLowering.cpp.

References Addr, llvm::EVT::changeVectorElementType(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::MemSDNode::getPointerInfo(), llvm::SelectionDAG::getStore(), llvm::MaskedStoreSDNode::getValue(), llvm::EVT::getVectorElementType(), llvm::MachinePointerInfo::getWithOffset(), and llvm::Offset.

Referenced by combineMaskedStore().

reduceVMULWidth()

static SDValue reduceVMULWidth ( SDNode *  N, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code.

Two typical patterns are handled: Pattern1: %2 = sext/zext <N x i8> %1 to <N x i32> %4 = sext/zext <N x i8> %3 to <N x i32> %5 = mul <N x i32> %2, %4

Pattern2: %2 = zext/sext <N x i16> %1 to <N x i32> %4 = zext/sext <N x i16> %3 to <N x i32> or %4 = build_vector <N x i32> C1, ..., CN (C1..CN are constants) %5 = mul <N x i32> %2, %4

There are four mul shrinking modes: If %2 == sext32(trunc8(%2)), i.e., the scalar value range of %2 is -128 to 128, and the scalar value range of %4 is also -128 to 128, generate pmullw+sext32 for it (MULS8 mode). If %2 == zext32(trunc8(%2)), i.e., the scalar value range of %2 is 0 to 255, and the scalar value range of %4 is also 0 to 255, generate pmullw+zext32 for it (MULU8 mode). If %2 == sext32(trunc16(%2)), i.e., the scalar value range of %2 is -32768 to 32767, and the scalar value range of %4 is also -32768 to 32767, generate pmullw+pmulhw for it (MULS16 mode). If %2 == zext32(trunc16(%2)), i.e., the scalar value range of %2 is 0 to 65535, and the scalar value range of %4 is also 0 to 65535, generate pmullw+pmulhuw for it (MULU16 mode).

Definition at line 48594 of file X86ISelLowering.cpp.

References canReduceVMulWidth(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::Function::hasMinSize(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, N, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by combineMul().

resolveTargetShuffleFromZeroables()

static void resolveTargetShuffleFromZeroables ( SmallVectorImpl< int > &  Mask, const APInt &  KnownUndef, const APInt &  KnownZero, bool  ResolveKnownZeros = true  )

static

Definition at line 5872 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::getBitWidth(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineX86ShufflesRecursively(), and getTargetShuffleInputs().

resolveTargetShuffleInputsAndMask()

static void resolveTargetShuffleInputsAndMask ( SmallVectorImpl< SDValue > &  Inputs, SmallVectorImpl< int > &  Mask  )

static

Removes unused/repeated shuffle source inputs and adjusts the shuffle mask.

Definition at line 6440 of file X86ISelLowering.cpp.

References isUndef(), llvm::none_of(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorBase< Size_T >::size(), and llvm::SM_SentinelUndef.

Referenced by combineHorizOpWithShuffle(), combineX86ShuffleChainWithExtract(), combineX86ShufflesRecursively(), and isHorizontalBinOp().

resolveZeroablesFromTargetShuffle()

static void resolveZeroablesFromTargetShuffle ( const SmallVectorImpl< int > &  Mask, APInt &  KnownUndef, APInt &  KnownZero  )

static

Definition at line 5889 of file X86ISelLowering.cpp.

References llvm::APInt::getZero(), llvm::APInt::setBit(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineX86ShuffleChain(), and getTargetShuffleInputs().

scalarizeExtEltFP()

static SDValue scalarizeExtEltFP ( SDNode *  ExtElt, SelectionDAG &  DAG, const X86Subtarget &  Subtarget, TargetLowering::DAGCombinerInfo &  DCI  )

static

Extracting a scalar FP value from vector element 0 is free, so extract each operand first, then perform the math as a scalar op.

Definition at line 45905 of file X86ISelLowering.cpp.

References assert(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FABS, llvm::ISD::FADD, llvm::ISD::FCEIL, llvm::ISD::FCOPYSIGN, llvm::ISD::FDIV, llvm::ISD::FFLOOR, llvm::ISD::FMA, llvm::ISD::FMAD, llvm::X86ISD::FMAX, llvm::ISD::FMAXIMUM, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::X86ISD::FMIN, llvm::ISD::FMINIMUM, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm::ISD::FMUL, llvm::ISD::FNEARBYINT, llvm::X86ISD::FRCP, llvm::ISD::FREM, llvm::ISD::FRINT, llvm::ISD::FROUND, llvm::ISD::FROUNDEVEN, llvm::X86ISD::FRSQRT, llvm::ISD::FSQRT, llvm::ISD::FSUB, llvm::ISD::FTRUNC, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::isNullConstant(), llvm_unreachable, llvm::SDNode::ops(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SELECT, llvm::ISD::SETCC, and llvm::ISD::VSELECT.

Referenced by combineExtractVectorElt().

scalarizeVectorStore()

static SDValue scalarizeVectorStore ( StoreSDNode *  Store, MVT  StoreVT, SelectionDAG &  DAG  )

static

Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.

Definition at line 25065 of file X86ISelLowering.cpp.

References assert(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getStore(), llvm::MVT::getStoreSize(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::EVT::is128BitVector(), llvm::MVT::is128BitVector(), llvm::Offset, Ptr, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::TokenFactor.

Referenced by combineStore(), and llvm::TargetLowering::expandUnalignedStore().

scaleShuffleElements()

static bool scaleShuffleElements ( ArrayRef< int >  Mask, unsigned  NumDstElts, SmallVectorImpl< int > &  ScaledMask  )

static

Definition at line 3849 of file X86ISelLowering.cpp.

References assert(), canWidenShuffleElements(), llvm::narrowShuffleMaskElts(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by canonicalizeShuffleMaskWithHorizOp(), canScaleShuffleElements(), combineBlendOfPermutes(), combineEXTRACT_SUBVECTOR(), combineHorizOpWithShuffle(), combineTargetShuffle(), combineX86ShuffleChain(), and isHorizontalBinOp().

shouldExpandCmpArithRMWInIR()

static bool shouldExpandCmpArithRMWInIR ( AtomicRMWInst *  AI )

static

Definition at line 31478 of file X86ISelLowering.cpp.

References llvm::AtomicRMWInst::Add, llvm::AtomicRMWInst::And, llvm::User::getOperand(), llvm::AtomicRMWInst::getOperation(), llvm::Value::hasOneUse(), I, llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLT, llvm::PatternMatch::m_AllOnes(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_c_And(), llvm::PatternMatch::m_c_ICmp(), llvm::PatternMatch::m_c_Or(), llvm::PatternMatch::m_c_Xor(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_OneUse(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificICmp(), llvm::PatternMatch::m_Sub(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), llvm::AtomicRMWInst::Or, llvm::AtomicRMWInst::Sub, llvm::Instruction::user_back(), and llvm::AtomicRMWInst::Xor.

shouldUseHorizontalOp()

static bool shouldUseHorizontalOp ( bool  IsSingleSource, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Horizontal vector math instructions may be slower than normal math with shuffles.

Limit horizontal op codegen based on size/speed trade-offs, uarch implementation, and likely shuffle complexity of the alternate sequence.

Definition at line 19795 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::shouldOptForSize().

Referenced by canonicalizeShuffleMaskWithHorizOp(), combineArithReduction(), combineVectorHADDSUB(), isHorizontalBinOp(), lowerAddSubToHorizontalOp(), and LowerUINT_TO_FP_i64().

signExtendBitcastSrcVector()

static SDValue signExtendBitcastSrcVector ( SelectionDAG &  DAG, EVT  SExtVT, SDValue  Src, const SDLoc &  DL  )

static

Definition at line 44457 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, DL, llvm::ISD::FREEZE, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm_unreachable, llvm::ISD::OR, llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, signExtendBitcastSrcVector(), llvm::ISD::TRUNCATE, llvm::ISD::VSELECT, and llvm::ISD::XOR.

Referenced by combineBitcastvxi1(), and signExtendBitcastSrcVector().

SplitAndExtendv16i1()

static SDValue SplitAndExtendv16i1 ( unsigned  ExtOpc, MVT  VT, SDValue  In, const SDLoc &  dl, SelectionDAG &  DAG  )

static

Definition at line 20528 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::Hi, llvm::Lo, and llvm::ISD::TRUNCATE.

Referenced by LowerSIGN_EXTEND_Mask(), and LowerZERO_EXTEND_Mask().

splitAndLowerShuffle()

static SDValue splitAndLowerShuffle ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, SelectionDAG &  DAG, bool  SimpleOnly  )

static

Generic routine to split vector shuffle into half-sized shuffles.

This routine just extracts two subvectors, shuffles them independently, and then concatenates them back together. This should work effectively with all AVX vector shuffle types.

Definition at line 14764 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::Hi, llvm::Lo, llvm::peekThroughBitcasts(), and splitVector().

Referenced by lower256BitShuffle(), lower512BitShuffle(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsSplitOrBlend(), lowerV64I8Shuffle(), and lowerV8F32Shuffle().

SplitOpsAndApply()

template<typename F >

SDValue SplitOpsAndApply	(	SelectionDAG &	DAG,
		const X86Subtarget &	Subtarget,
		const SDLoc &	DL,
		EVT	VT,
		ArrayRef< SDValue >	Ops,
		F	Builder,
		bool	CheckBWI = true
	)

Definition at line 4323 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, extractSubVector(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::X86Subtarget::useAVX512Regs(), and llvm::X86Subtarget::useBWIRegs().

Referenced by combineArithReduction(), combineMulToPMADDWD(), combineMulToPMULDQ(), combineToHorizontalAddSub(), createPSADBW(), createVPDPBUSD(), detectPMADDUBSW(), matchPMADDWD(), matchPMADDWD_2(), and narrowVectorSelect().

splitVector()

static std::pair< SDValue, SDValue > splitVector ( SDValue  Op, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Definition at line 4245 of file X86ISelLowering.cpp.

References assert(), extractSubVector(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::SelectionDAG::isSplatValue(), and llvm::Lo.

Referenced by combineAndShuffleNot(), combineArithReduction(), combineMinMaxReduction(), LowerMULO(), splitAndLowerShuffle(), splitVectorOp(), splitVectorStore(), splitVSETCC(), and truncateVectorWithPACK().

splitVectorIntBinary()

static SDValue splitVectorIntBinary ( SDValue  Op, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Break a binary integer operation into 2 half sized ops and then concatenate the result back.

Definition at line 4305 of file X86ISelLowering.cpp.

References assert(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), and splitVectorOp().

Referenced by combineEXTRACT_SUBVECTOR(), LowerABD(), LowerADDSAT_SUBSAT(), lowerAddSub(), LowerAVG(), LowerMINMAX(), LowerMUL(), LowerMULH(), LowerRotate(), and LowerShift().

splitVectorIntUnary()

static SDValue splitVectorIntUnary ( SDValue  Op, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Break an unary integer operation into 2 half sized ops and then concatenate the result back.

Definition at line 4289 of file X86ISelLowering.cpp.

References assert(), llvm::EVT::getVectorNumElements(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), and splitVectorOp().

Referenced by LowerABS(), LowerAVXExtend(), LowerBITREVERSE(), LowerBITREVERSE_XOP(), LowerSIGN_EXTEND(), LowerVectorCTLZ(), LowerVectorCTLZ_AVX512CDI(), and LowerVectorCTPOP().

splitVectorOp()

static SDValue splitVectorOp ( SDValue  Op, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Break an operation into 2 half sized ops and then concatenate the results.

Definition at line 4264 of file X86ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::GetSplitDestVTs(), I, and splitVector().

Referenced by LowerFunnelShift(), splitVectorIntBinary(), and splitVectorIntUnary().

splitVectorStore()

static SDValue splitVectorStore ( StoreSDNode *  Store, SelectionDAG &  DAG  )

static

Change a vector store into a pair of half-size vector stores.

Definition at line 25031 of file X86ISelLowering.cpp.

References assert(), DL, llvm::TypeSize::getFixed(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStore(), llvm::EVT::getStoreSize(), llvm::SDValue::getValueType(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), splitVector(), and llvm::ISD::TokenFactor.

Referenced by combineStore(), and LowerStore().

splitVSETCC()

static SDValue splitVSETCC ( EVT  VT, SDValue  LHS, SDValue  RHS, ISD::CondCode  Cond, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Break a VSETCC 256/512-bit vector into two new 128/256 ones and then concatenate the result back.

Definition at line 23425 of file X86ISelLowering.cpp.

References assert(), CC, llvm::ISD::CONCAT_VECTORS, Cond, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::EVT::isInteger(), LHS, RHS, llvm::ISD::SETCC, and splitVector().

Referenced by LowerVSETCC().

supportedVectorShiftWithBaseAmnt()

static bool supportedVectorShiftWithBaseAmnt ( EVT  VT, const X86Subtarget &  Subtarget, unsigned  Opcode  )

static

Definition at line 29693 of file X86ISelLowering.cpp.

References supportedVectorShiftWithImm().

Referenced by LowerFunnelShift(), and LowerShiftByScalarVariable().

supportedVectorShiftWithImm()

static bool supportedVectorShiftWithImm ( EVT  VT, const X86Subtarget &  Subtarget, unsigned  Opcode  )

static

Definition at line 29664 of file X86ISelLowering.cpp.

References assert(), llvm::EVT::getScalarSizeInBits(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::EVT::isSimple(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and llvm::X86Subtarget::useAVX512Regs().

Referenced by combineAndMaskToShift(), combineScalarToVector(), LowerFunnelShift(), LowerRotate(), LowerShiftByScalarImmediate(), and supportedVectorShiftWithBaseAmnt().

supportedVectorVarShift()

static bool supportedVectorVarShift ( EVT  VT, const X86Subtarget &  Subtarget, unsigned  Opcode  )

static

Definition at line 29700 of file X86ISelLowering.cpp.

References assert(), llvm::EVT::getScalarSizeInBits(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::EVT::isSimple(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and llvm::X86Subtarget::useAVX512Regs().

Referenced by combineSelect(), combineShiftLeft(), combineShiftRightArithmetic(), combineShiftRightLogical(), LowerFunnelShift(), LowerRotate(), and LowerShift().

TranslateIntegerX86CC()

static X86::CondCode TranslateIntegerX86CC ( ISD::CondCode  SetCCOpcode )

static

Definition at line 2921 of file X86ISelLowering.cpp.

References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, and llvm::ISD::SETULT.

Referenced by TranslateX86CC().

TranslateX86CC()

static X86::CondCode TranslateX86CC ( ISD::CondCode  SetCCOpcode, const SDLoc &  DL, bool  isFP, SDValue &  LHS, SDValue &  RHS, SelectionDAG &  DAG  )

static

Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make.

Definition at line 2942 of file X86ISelLowering.cpp.

References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_INVALID, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_NS, llvm::X86::COND_P, llvm::X86::COND_S, DL, llvm::SelectionDAG::getConstant(), llvm::ISD::isNON_EXTLoad(), LHS, llvm_unreachable, RHS, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, std::swap(), and TranslateIntegerX86CC().

translateX86FSETCC()

static unsigned translateX86FSETCC ( ISD::CondCode  SetCCOpcode, SDValue &  Op0, SDValue &  Op1, bool &  IsAlwaysSignaling  )

static

Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs.

Definition at line 23362 of file X86ISelLowering.cpp.

References llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, and std::swap().

Referenced by combineConcatVectorOps(), and LowerVSETCC().

truncateAVX512SetCCNoBWI()

static SDValue truncateAVX512SetCCNoBWI ( EVT  VT, EVT  OpVT, SDValue  LHS, SDValue  RHS, ISD::CondCode  CC, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just pre-promote its result type since vXi1 vectors don't get promoted during type legalization.

Definition at line 55291 of file X86ISelLowering.cpp.

References CC, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX512(), llvm::EVT::isVector(), LHS, RHS, and llvm::ISD::TRUNCATE.

Referenced by combineSetCC().

truncateVectorWithPACK()

static SDValue truncateVectorWithPACK ( unsigned  Opcode, EVT  DstVT, SDValue  In, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Helper to recursively truncate vector elements in half with PACKSS/PACKUS.

It makes use of the fact that vectors with enough leading sign/zero bits prevent the PACKSS/PACKUS from saturating the results. AVX2 (Int256) sub-targets require extra shuffling as the PACK*S operates within each 128-bit lane.

Definition at line 20616 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), LHS, llvm::Lo, llvm::narrowShuffleMaskElts(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, RHS, splitVector(), truncateVectorWithPACK(), and widenSubVector().

Referenced by combineTruncateWithSat(), LowerTruncateVecPackWithSignBits(), llvm::X86TargetLowering::ReplaceNodeResults(), truncateVectorWithPACK(), truncateVectorWithPACKSS(), and truncateVectorWithPACKUS().

truncateVectorWithPACKSS()

static SDValue truncateVectorWithPACKSS ( EVT  DstVT, SDValue  In, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Truncate using inreg sign extension and X86ISD::PACKSS.

Definition at line 20746 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getValueType(), llvm::X86ISD::PACKSS, llvm::ISD::SIGN_EXTEND_INREG, and truncateVectorWithPACK().

Referenced by LowerTruncateVecPack().

truncateVectorWithPACKUS()

static SDValue truncateVectorWithPACKUS ( EVT  DstVT, SDValue  In, const SDLoc &  DL, const X86Subtarget &  Subtarget, SelectionDAG &  DAG  )

static

Truncate using inreg zero extension (AND mask) and X86ISD::PACKUS.

e.g. trunc <8 x i32> X to <8 x i16> --> MaskX = X & 0xffff (clear high bits to prevent saturation) packus (extract_subv MaskX, 0), (extract_subv MaskX, 1)

Definition at line 20738 of file X86ISelLowering.cpp.

References DL, llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86ISD::PACKUS, and truncateVectorWithPACK().

Referenced by LowerTruncateVecPack().

useEGPRInlineAsm()

static bool useEGPRInlineAsm ( const X86Subtarget &  Subtarget )

static

Definition at line 60118 of file X86ISelLowering.cpp.

Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().

useVectorCast()

static bool useVectorCast ( unsigned  Opcode, MVT  FromVT, MVT  ToVT, const X86Subtarget &  Subtarget  )

static

Definition at line 19411 of file X86ISelLowering.cpp.

References llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::SINT_TO_FP, and llvm::ISD::UINT_TO_FP.

Referenced by vectorizeExtractedCast().

useVPTERNLOG()

static bool useVPTERNLOG ( const X86Subtarget &  Subtarget, MVT  VT  )

static

Definition at line 3041 of file X86ISelLowering.cpp.

References llvm::X86Subtarget::canExtendTo512DQ(), and llvm::MVT::is512BitVector().

Referenced by canonicalizeBitSelect(), LowerADDSAT_SUBSAT(), LowerFunnelShift(), and LowerRotate().

vectorizeExtractedCast()

static SDValue vectorizeExtractedCast ( SDValue  Cast, const SDLoc &  DL, SelectionDAG &  DAG, const X86Subtarget &  Subtarget  )

static

Given a scalar cast operation that is extracted from a vector, try to vectorize the cast op followed by extraction.

This will avoid an expensive round-trip between XMM and GPR.

Definition at line 19436 of file X86ISelLowering.cpp.

References DL, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::isNullConstant(), and useVectorCast().

widenMaskVector()

static SDValue widenMaskVector ( SDValue  Vec, bool  ZeroNewElements, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.

Definition at line 4145 of file X86ISelLowering.cpp.

References llvm::SDValue::getSimpleValueType(), widenMaskVectorType(), and widenSubVector().

Referenced by combineCMP(), ExtractBitFromMaskVector(), lower1BitShuffle(), lower1BitShuffleAsKSHIFTR(), and LowerEXTRACT_SUBVECTOR().

widenMaskVectorType()

static MVT widenMaskVectorType ( MVT  VT, const X86Subtarget &  Subtarget  )

static

Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.

Definition at line 4135 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getVectorElementType(), and llvm::MVT::getVectorNumElements().

Referenced by insert1BitVector(), LowerCONCAT_VECTORSvXi1(), and widenMaskVector().

widenSubVector() [1/2]

static SDValue widenSubVector ( MVT  VT, SDValue  Vec, bool  ZeroNewElements, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, const SDLoc &  dl  )

static

Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.

Definition at line 4107 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getFixedSizeInBits(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), getZeroVector(), and llvm::ISD::INSERT_SUBVECTOR.

Referenced by canonicalizeShuffleMaskWithHorizOp(), combineBitcastvxi1(), combineConcatVectorOps(), combineEXTRACT_SUBVECTOR(), combineTargetShuffle(), combineX86ShuffleChain(), combineX86ShufflesRecursively(), createVariablePermute(), getAVX512Node(), getAVX512TruncNode(), LowerCONCAT_VECTORSvXi1(), lowerShuffleWithPERMV(), LowerTruncateVecPack(), LowerTruncateVecPackWithSignBits(), lowerVECTOR_COMPRESS(), llvm::X86TargetLowering::ReplaceNodeResults(), llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(), truncateVectorWithPACK(), widenMaskVector(), and widenSubVector().

widenSubVector() [2/2]

static SDValue widenSubVector ( SDValue  Vec, bool  ZeroNewElements, const X86Subtarget &  Subtarget, SelectionDAG &  DAG, const SDLoc &  dl, unsigned  WideSizeInBits  )

static

Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.

Definition at line 4121 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorVT(), and widenSubVector().

Variable Documentation

BrMergingBaseCostThresh

cl::opt< int > BrMergingBaseCostThresh("x86-br-merging-base-cost", cl::init(2), cl::desc( "Sets the cost threshold for when multiple conditionals will be merged " "into one branch versus be split in multiple branches. Merging " "conditionals saves branches at the cost of additional instructions. " "This value sets the instruction cost limit, below which conditionals " "will be merged, and above which conditionals will be split. Set to -1 " "to never merge branches."), cl::Hidden) ( "x86-br-merging-base-cost"  , cl::init(2)  , cl::desc( "Sets the cost threshold for when multiple conditionals will be merged " "into one branch versus be split in multiple branches. Merging " "conditionals saves branches at the cost of additional instructions. " "This value sets the instruction cost limit, below which conditionals " "will be merged, and above which conditionals will be split. Set to -1 " "to never merge branches.")  , cl::Hidden    )

static

Referenced by llvm::X86TargetLowering::getJumpConditionMergingParams().

BrMergingCcmpBias

cl::opt< int > BrMergingCcmpBias("x86-br-merging-ccmp-bias", cl::init(6), cl::desc("Increases 'x86-br-merging-base-cost' in cases that the target " "supports conditional compare instructions."), cl::Hidden) ( "x86-br-merging-ccmp-bias"  , cl::init(6)  , cl::desc("Increases 'x86-br-merging-base-cost' in cases that the target " "supports conditional compare instructions.")  , cl::Hidden    )

static

Referenced by llvm::X86TargetLowering::getJumpConditionMergingParams().

BrMergingLikelyBias

cl::opt< int > BrMergingLikelyBias("x86-br-merging-likely-bias", cl::init(0), cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "likely, then it is likely that if the conditionals are split " "both sides will be executed, so it may be desirable to increase " "the instruction cost threshold. Set to -1 to never merge likely " "branches."), cl::Hidden) ( "x86-br-merging-likely-bias"  , cl::init(0)  , cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "likely, then it is likely that if the conditionals are split " "both sides will be executed, so it may be desirable to increase " "the instruction cost threshold. Set to -1 to never merge likely " "branches.")  , cl::Hidden    )

static

Referenced by llvm::X86TargetLowering::getJumpConditionMergingParams().

BrMergingUnlikelyBias

cl::opt< int > BrMergingUnlikelyBias("x86-br-merging-unlikely-bias", cl::init(-1), cl::desc( "Decreases 'x86-br-merging-base-cost' in cases that it is unlikely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "unlikely, then it is unlikely that if the conditionals are split " "both sides will be executed, so it may be desirable to decrease " "the instruction cost threshold. Set to -1 to never merge unlikely " "branches."), cl::Hidden) ( "x86-br-merging-unlikely-bias"  , cl::init(-1)  , cl::desc( "Decreases 'x86-br-merging-base-cost' in cases that it is unlikely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "unlikely, then it is unlikely that if the conditionals are split " "both sides will be executed, so it may be desirable to decrease " "the instruction cost threshold. Set to -1 to never merge unlikely " "branches.")  , cl::Hidden    )

static

Referenced by llvm::X86TargetLowering::getJumpConditionMergingParams().

ExperimentalPrefInnermostLoopAlignment

cl::opt< int > ExperimentalPrefInnermostLoopAlignment("x86-experimental-pref-innermost-loop-alignment", cl::init(4), cl::desc( "Sets the preferable loop alignment for experiments (as log2 bytes) " "for innermost loops only. If specified, this option overrides " "alignment set by x86-experimental-pref-loop-alignment."), cl::Hidden) ( "x86-experimental-pref-innermost-loop-alignment"  , cl::init(4)  , cl::desc( "Sets the preferable loop alignment for experiments (as log2 bytes) " "for innermost loops only. If specified, this option overrides " "alignment set by x86-experimental-pref-loop-alignment.")  , cl::Hidden    )

static

Referenced by llvm::X86TargetLowering::getPrefLoopAlignment().

FPStateSize

const unsigned FPStateSize = 32

Definition at line 28325 of file X86ISelLowering.cpp.

FPStateSizeInBits

const unsigned FPStateSizeInBits = FPStateSize * 8

Definition at line 28326 of file X86ISelLowering.cpp.

MulConstantOptimization

cl::opt< bool > MulConstantOptimization("mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc."), cl::Hidden) ( "mul-constant-optimization"  , cl::init(true)  , cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc.")  , cl::Hidden    )

static

Referenced by combineMul().

WidenShift

cl::opt< bool > WidenShift("x86-widen-shift", cl::init(true), cl::desc("Replacte narrow shifts with wider shifts."), cl::Hidden) ( "x86-widen-shift"  , cl::init(true)  , cl::desc("Replacte narrow shifts with wider shifts.")  , cl::Hidden    )

static

Referenced by LowerShift().

X87StateSize

const unsigned X87StateSize = 28

Definition at line 28324 of file X86ISelLowering.cpp.

Referenced by createSetFPEnvNodes().