LLVM  4.0.0
Macros | Enumerations | Functions | Variables
X86ISelLowering.cpp File Reference
#include "X86ISelLowering.h"
#include "Utils/X86ShuffleDecode.h"
#include "X86CallingConv.h"
#include "X86FrameLowering.h"
#include "X86InstrBuilder.h"
#include "X86IntrinsicsInfo.h"
#include "X86MachineFunctionInfo.h"
#include "X86ShuffleDecodeConstantPool.h"
#include "X86TargetMachine.h"
#include "X86TargetObjectFile.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Analysis/EHPersonalities.h"
#include "llvm/CodeGen/IntrinsicLowering.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <bitset>
#include <cctype>
#include <numeric>
#include "X86GenCallingConv.inc"
Include dependency graph for X86ISelLowering.cpp:

Go to the source code of this file.

Macros

#define DEBUG_TYPE   "x86-isel"
 

Enumerations

enum  StructReturnType { NotStructReturn, RegStructReturn, StackStructReturn }
 CallIsStructReturn - Determines whether a call uses struct return semantics. More...
 
enum  ShrinkMode { MULS8, MULU8, MULS16, MULU16 }
 Different mul shrinking modes. More...
 

Functions

 STATISTIC (NumTailCalls,"Number of tail calls")
 
static void getMaxByValAlign (Type *Ty, unsigned &MaxAlign)
 Helper for getByValTypeAlignment to determine the desired ByVal argument alignment. More...
 
static SDValue lowerMasksToReg (const SDValue &ValArg, const EVT &ValLoc, const SDLoc &Dl, SelectionDAG &DAG)
 Lowers masks values (v*i1) to the local register values. More...
 
static void Passv64i1ArgInRegs (const SDLoc &Dl, SelectionDAG &DAG, SDValue Chain, SDValue &Arg, SmallVector< std::pair< unsigned, SDValue >, 8 > &RegsToPass, CCValAssign &VA, CCValAssign &NextVA, const X86Subtarget &Subtarget)
 Breaks v64i1 value into two registers and adds the new node to the DAG. More...
 
static SDValue getv64i1Argument (CCValAssign &VA, CCValAssign &NextVA, SDValue &Root, SelectionDAG &DAG, const SDLoc &Dl, const X86Subtarget &Subtarget, SDValue *InFlag=nullptr)
 Reads two 32 bit registers and creates a 64 bit mask value. More...
 
static SDValue lowerRegToMasks (const SDValue &ValArg, const EVT &ValVT, const EVT &ValLoc, const SDLoc &Dl, SelectionDAG &DAG)
 The function will lower a register of various sizes (8/16/32/64) to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1) More...
 
static StructReturnType callIsStructReturn (const SmallVectorImpl< ISD::OutputArg > &Outs, bool IsMCU)
 
static StructReturnType argsAreStructReturn (const SmallVectorImpl< ISD::InputArg > &Ins, bool IsMCU)
 Determines whether a function uses struct return semantics. More...
 
static SDValue CreateCopyOfByValArgument (SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl)
 Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute. More...
 
static bool canGuaranteeTCO (CallingConv::ID CC)
 Return true if the calling convention is one that we can guarantee TCO for. More...
 
static bool mayTailCallThisCC (CallingConv::ID CC)
 Return true if we might ever do TCO for calls with this calling convention. More...
 
static bool shouldGuaranteeTCO (CallingConv::ID CC, bool GuaranteedTailCallOpt)
 Return true if the function is being made into a tailcall target by changing its ABI. More...
 
static ArrayRef< MCPhysRegget64BitArgumentGPRs (CallingConv::ID CallConv, const X86Subtarget &Subtarget)
 
static ArrayRef< MCPhysRegget64BitArgumentXMMs (MachineFunction &MF, CallingConv::ID CallConv, const X86Subtarget &Subtarget)
 
static bool isSortedByValueNo (const SmallVectorImpl< CCValAssign > &ArgLocs)
 
static SDValue EmitTailCallStoreRetAddr (SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue RetAddrFrIdx, EVT PtrVT, unsigned SlotSize, int FPDiff, const SDLoc &dl)
 Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0). More...
 
static SDValue getMOVL (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2)
 Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width. More...
 
static bool MatchingStackOffset (SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const X86InstrInfo *TII, const CCValAssign &VA)
 Return true if the given stack call argument is already available in the same position (relatively) of the caller's incoming argument stack. More...
 
static bool MayFoldLoad (SDValue Op)
 
static bool MayFoldIntoStore (SDValue Op)
 
static bool MayFoldIntoZeroExtend (SDValue Op)
 
static bool isTargetShuffle (unsigned Opcode)
 
static bool isTargetShuffleVariableMask (unsigned Opcode)
 
static bool isX86CCUnsigned (unsigned X86CC)
 Return true if the condition is an unsigned comparison operation. More...
 
static X86::CondCode TranslateIntegerX86CC (ISD::CondCode SetCCOpcode)
 
static X86::CondCode TranslateX86CC (ISD::CondCode SetCCOpcode, const SDLoc &DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG)
 Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make. More...
 
static bool hasFPCMov (unsigned X86CC)
 Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. More...
 
static bool isUndefOrEqual (int Val, int CmpVal)
 Val is the undef sentinel value or equal to the specified value. More...
 
static bool isUndefOrZero (int Val)
 Val is either the undef or zero sentinel value. More...
 
static bool isUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value. More...
 
static bool isUndefOrInRange (int Val, int Low, int Hi)
 Return true if Val is undef or if its value falls within the specified range (L, H]. More...
 
static bool isUndefOrInRange (ArrayRef< int > Mask, int Low, int Hi)
 Return true if every element in Mask is undef or if its value falls within the specified range (L, H]. More...
 
static bool isUndefOrZeroOrInRange (int Val, int Low, int Hi)
 Return true if Val is undef, zero or if its value falls within the specified range (L, H]. More...
 
static bool isUndefOrZeroOrInRange (ArrayRef< int > Mask, int Low, int Hi)
 Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H]. More...
 
static bool isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size]. More...
 
static bool isSequentialOrUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero. More...
 
static bool isUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero. More...
 
static bool canWidenShuffleElements (ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask)
 Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled. More...
 
static void scaleShuffleMask (int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
 Helper function to scale a shuffle or target shuffle mask, replacing each mask index with the scaled sequential indices for an equivalent narrowed mask. More...
 
static bool isVEXTRACTIndex (SDNode *N, unsigned vecWidth)
 Return true if the specified EXTRACT_SUBVECTOR operand specifies a vector extract that is suitable for instruction that extract 128 or 256 bit vectors. More...
 
static bool isVINSERTIndex (SDNode *N, unsigned vecWidth)
 Return true if the specified INSERT_SUBVECTOR operand specifies a subvector insert that is suitable for input to insertion of 128 or 256-bit subvectors. More...
 
static unsigned getExtractVEXTRACTImmediate (SDNode *N, unsigned vecWidth)
 
static unsigned getInsertVINSERTImmediate (SDNode *N, unsigned vecWidth)
 
static SDValue getConstVector (ArrayRef< int > Values, MVT VT, SelectionDAG &DAG, const SDLoc &dl, bool IsMask=false)
 
static SDValue getConstVector (ArrayRef< APInt > Bits, SmallBitVector &Undefs, MVT VT, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue getZeroVector (MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Returns a vector of specified type with all zero elements. More...
 
static SDValue extractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
 
static SDValue extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to grab 128-bits from a vector > 128 bits. More...
 
static SDValue extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to grab 256-bits from a 512-bit vector. More...
 
static SDValue insertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
 
static SDValue insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to put 128-bits into a vector > 128 bits. More...
 
static SDValue insert256BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue insert1BitVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Insert i1-subvector to i1-vector. More...
 
static SDValue concat128BitVectors (SDValue V1, SDValue V2, EVT VT, unsigned NumElems, SelectionDAG &DAG, const SDLoc &dl)
 Concat two 128-bit vectors into a 256 bit vector using VINSERTF128 instructions. More...
 
static SDValue concat256BitVectors (SDValue V1, SDValue V2, EVT VT, unsigned NumElems, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue getOnesVector (EVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Returns a vector of specified type with all bits set. More...
 
static void createUnpackShuffleMask (MVT VT, SmallVectorImpl< int > &Mask, bool Lo, bool Unary)
 Generate unpacklo/unpackhi shuffle mask. More...
 
static SDValue getUnpackl (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2)
 Returns a vector_shuffle node for an unpackl operation. More...
 
static SDValue getUnpackh (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2)
 Returns a vector_shuffle node for an unpackh operation. More...
 
static SDValue getShuffleVectorZeroOrUndef (SDValue V2, int Idx, bool IsZero, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Return a vector_shuffle of the specified vector of zero or undef vector. More...
 
static SDValue peekThroughBitcasts (SDValue V)
 
static SDValue peekThroughOneUseBitcasts (SDValue V)
 
static const ConstantgetTargetConstantFromNode (SDValue Op)
 
static bool getTargetConstantBitsFromNode (SDValue Op, unsigned EltSizeInBits, SmallBitVector &UndefElts, SmallVectorImpl< APInt > &EltBits)
 
static bool getTargetShuffleMaskIndices (SDValue MaskNode, unsigned MaskEltSizeInBits, SmallVectorImpl< uint64_t > &RawMask)
 
static bool getTargetShuffleMask (SDNode *N, MVT VT, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask, bool &IsUnary)
 Calculates the shuffle mask corresponding to the target-specific opcode. More...
 
static bool setTargetShuffleZeroElements (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops)
 Check a target shuffle mask's inputs to see if we can set any values to SM_SentinelZero - this is for elements that are known to be zero (not just zeroable) from their inputs. More...
 
static bool getFauxShuffleMask (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops)
 
static bool resolveTargetShuffleInputs (SDValue Op, SDValue &Op0, SDValue &Op1, SmallVectorImpl< int > &Mask)
 Calls setTargetShuffleZeroElements to resolve a target shuffle mask's inputs and set the SM_SentinelUndef and SM_SentinelZero values. More...
 
static SDValue getShuffleScalarElt (SDNode *N, unsigned Index, SelectionDAG &DAG, unsigned Depth)
 Returns the scalar element that will make up the ith element of the result of the vector shuffle. More...
 
static SDValue LowerBuildVectorv16i8 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget, const TargetLowering &TLI)
 Custom lower build_vector of v16i8. More...
 
static SDValue LowerBuildVectorv8i16 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget, const TargetLowering &TLI)
 Custom lower build_vector of v8i16. More...
 
static SDValue LowerBuildVectorv4x32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget, const TargetLowering &TLI)
 Custom lower build_vector of v4i32 or v4f32. More...
 
static SDValue getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &dl)
 Return a vector logical shift node. More...
 
static SDValue LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, const SDLoc &dl, SelectionDAG &DAG)
 
static SDValue EltsFromConsecutiveLoads (EVT VT, ArrayRef< SDValue > Elts, SDLoc &DL, SelectionDAG &DAG, bool isAfterLegalize)
 Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'. More...
 
static ConstantgetConstantVector (MVT VT, APInt SplatValue, unsigned SplatBitSize, LLVMContext &C)
 
static bool isUseOfShuffle (SDNode *N)
 
static SDValue LowerVectorBroadcast (BuildVectorSDNode *BVOp, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Attempt to use the vbroadcast instruction to generate a splat value for the following cases: More...
 
static int getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx)
 For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index. More...
 
static SDValue buildFromShuffleMostly (SDValue Op, SelectionDAG &DAG)
 
static SDValue ConvertI1VectorToInteger (SDValue Op, SelectionDAG &DAG)
 
static bool isHorizontalBinOp (const BuildVectorSDNode *N, unsigned Opcode, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1)
 Return true if N implements a horizontal binop and return the operands for the horizontal binop into V0 and V1. More...
 
static SDValue ExpandHorizontalBinOp (const SDValue &V0, const SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI)
 Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector. More...
 
static bool isAddSub (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1)
 Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB operation. More...
 
static bool isFMAddSub (const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2)
 Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB(Opnd0, Opnd1) into FMADDSUB(x, y, Opnd1). More...
 
static SDValue lowerToAddSubOrFMAddSub (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB node. More...
 
static SDValue LowerToHorizontalOp (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower BUILD_VECTOR to a horizontal add/sub operation if possible. More...
 
static SDValue lowerBuildVectorToBitOp (BuildVectorSDNode *Op, SelectionDAG &DAG)
 If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors. More...
 
static SDValue materializeVectorConstant (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Create a vector constant without a load. More...
 
static SDValue LowerAVXCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerCONCAT_VECTORSvXi1 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCONCAT_VECTORS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool isNoopShuffleMask (ArrayRef< int > Mask)
 Tiny helper function to identify a no-op mask. More...
 
static bool is128BitLaneCrossingShuffleMask (MVT VT, ArrayRef< int > Mask)
 Test whether there are elements crossing 128-bit lanes in this shuffle mask. More...
 
static bool isRepeatedShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each sub-lane. More...
 
static bool is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each 128-bit lane. More...
 
static bool is256BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each 256-bit lane. More...
 
static bool isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a target shuffle mask is equivalent within each sub-lane. More...
 
static bool isShuffleEquivalent (SDValue V1, SDValue V2, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask)
 Checks whether a shuffle mask is equivalent to an explicit list of arguments. More...
 
static bool isTargetShuffleEquivalent (ArrayRef< int > Mask, ArrayRef< int > ExpectedMask)
 Checks whether a target shuffle mask is equivalent to an explicit pattern. More...
 
static unsigned getV4X86ShuffleImm (ArrayRef< int > Mask)
 Get a 4-lane 8-bit shuffle immediate for a mask. More...
 
static SDValue getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, SDLoc DL, SelectionDAG &DAG)
 
static SmallBitVector computeZeroableShuffleElements (ArrayRef< int > Mask, SDValue V1, SDValue V2)
 Compute whether each element of a shuffle is zeroable. More...
 
static bool isNonZeroElementsInOrder (const SmallBitVector Zeroable, ArrayRef< int > Mask, const EVT &VectorType, bool &IsZeroSideLeft)
 
static SDValue lowerVectorShuffleWithPSHUFB (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a shuffle with a single PSHUFB of V1 or V2. More...
 
static SDValue getMaskNode (SDValue Mask, MVT MaskVT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics. More...
 
static unsigned convertBitVectorToUnsiged (const SmallBitVector &Zeroable)
 
static SDValue lowerVectorShuffleToEXPAND (const SDLoc &DL, MVT VT, const SmallBitVector &Zeroable, ArrayRef< int > Mask, SDValue &V1, SDValue &V2, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerVectorShuffleWithUNPCK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 
static SDValue lowerVectorShuffleAsBitMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SelectionDAG &DAG)
 Try to emit a bitmask instruction for a shuffle. More...
 
static SDValue lowerVectorShuffleAsBitBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to emit a blend instruction for a shuffle using bit math. More...
 
static SDValue lowerVectorShuffleAsBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Original, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to emit a blend instruction for a shuffle. More...
 
static SDValue lowerVectorShuffleAsBlendAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to lower as a blend of elements from two inputs followed by a single-input permutation. More...
 
static SDValue lowerVectorShuffleAsDecomposedShuffleBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Generic routine to decompose a shuffle and blend into indepndent blends and permutes. More...
 
static int matchVectorShuffleAsRotate (SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
 Try to lower a vector shuffle as a rotation. More...
 
static int matchVectorShuffleAsByteRotate (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
 Try to lower a vector shuffle as a byte rotation. More...
 
static SDValue lowerVectorShuffleAsByteRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerVectorShuffleAsRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a dword/qword rotation. More...
 
static int matchVectorShuffleAsShift (MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget)
 Try to lower a vector shuffle as a bit shift (shifts in zeros). More...
 
static SDValue lowerVectorShuffleAsShift (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerVectorShuffleWithSSE4A (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SelectionDAG &DAG)
 Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ. More...
 
static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend (const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower a vector shuffle as a zero or any extension. More...
 
static SDValue lowerVectorShuffleAsZeroOrAnyExtend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a zero extension on any microarch. More...
 
static SDValue getScalarValueForVectorElement (SDValue V, int Idx, SelectionDAG &DAG)
 Try to get a scalar value for a specific element of a vector. More...
 
static bool isShuffleFoldableLoad (SDValue V)
 Helper to test for a load that can be folded with x86 shuffles. More...
 
static SDValue lowerVectorShuffleAsElementInsertion (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower insertion of a single element into a zero vector. More...
 
static SDValue lowerVectorShuffleAsTruncBroadcast (const SDLoc &DL, MVT VT, SDValue V0, int BroadcastIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements. More...
 
static SDValue lowerVectorShuffleAsBroadcast (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower broadcast of a single element. More...
 
static bool matchVectorShuffleAsInsertPS (SDValue &V1, SDValue &V2, unsigned &InsertPSMask, const SmallBitVector &Zeroable, ArrayRef< int > Mask, SelectionDAG &DAG)
 
static SDValue lowerVectorShuffleAsInsertPS (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SelectionDAG &DAG)
 
static SDValue lowerVectorShuffleAsPermuteAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction. More...
 
static SDValue lowerV2F64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 2-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV2I64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 2-lane 64-bit integer shuffles. More...
 
static bool isSingleSHUFPSMask (ArrayRef< int > Mask)
 Test whether this can be lowered with a single SHUFPS instruction. More...
 
static SDValue lowerVectorShuffleWithSHUFPS (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 Lower a vector shuffle using the SHUFPS instruction. More...
 
static SDValue lowerV4F32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower 4-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV4I32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower 4-lane i32 vector shuffles. More...
 
static SDValue lowerV8I16GeneralSingleInputVectorShuffle (const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part. More...
 
static SDValue lowerVectorShuffleAsBlendOfPSHUFBs (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse)
 Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used. More...
 
static SDValue lowerV8I16VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Generic lowering of 8-lane i16 shuffles. More...
 
static int canLowerByDroppingEvenElements (ArrayRef< int > Mask, bool IsSingleInput)
 Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped. More...
 
static SDValue lowerV16I8VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Generic lowering of v16i8 shuffles. More...
 
static SDValue lower128BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Dispatching routine to lower various 128-bit x86 vector shuffles. More...
 
static SDValue splitAndLowerVectorShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Generic routine to split vector shuffle into half-sized shuffles. More...
 
static SDValue lowerVectorShuffleAsSplitOrBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Either split a vector in halves or decompose the shuffles and the blend. More...
 
static SDValue lowerVectorShuffleAsLanePermuteAndBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Lower a vector shuffle crossing multiple 128-bit lanes as a permutation and blend of those lanes. More...
 
static SDValue lowerV2X128VectorShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering 2-lane 128-bit shuffles. More...
 
static SDValue lowerVectorShuffleByMerging128BitLanes (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane. More...
 
static SDValue lowerVectorShuffleWithUndefHalf (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower shuffles where an entire half of a 256-bit vector is UNDEF. More...
 
static bool isShuffleMaskInputInPlace (int Input, ArrayRef< int > Mask)
 Test whether the specified input (0 or 1) is in-place blended by the given mask. More...
 
static SDValue lowerShuffleAsRepeatedMaskAndLanePermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes. More...
 
static bool matchVectorShuffleWithSHUFPD (MVT VT, SDValue &V1, SDValue &V2, unsigned &ShuffleImm, ArrayRef< int > Mask)
 
static SDValue lowerVectorShuffleWithSHUFPD (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 
static SDValue lowerVectorShuffleWithPERMV (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 
static SDValue lowerV4F64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 4-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV4I64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 4-lane 64-bit integer shuffles. More...
 
static SDValue lowerV8F32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV8I32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 32-bit integer shuffles. More...
 
static SDValue lowerV16I16VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 16-bit integer shuffles. More...
 
static SDValue lowerV32I8VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 32-lane 8-bit integer shuffles. More...
 
static SDValue lower256BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 High-level routine to lower various 256-bit x86 vector shuffles. More...
 
static SDValue lowerV4X128VectorShuffle (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 Try to lower a vector shuffle as a 128-bit shuffles. More...
 
static SDValue lowerV8F64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV16F32VectorShuffle (SDLoc DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV8I64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 64-bit integer shuffles. More...
 
static SDValue lowerV16I32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 32-bit integer shuffles. More...
 
static SDValue lowerV32I16VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 32-lane 16-bit integer shuffles. More...
 
static SDValue lowerV64I8VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 64-lane 8-bit integer shuffles. More...
 
static SDValue lower512BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 High-level routine to lower various 512-bit x86 vector shuffles. More...
 
static SDValue lower1BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool canonicalizeShuffleMaskWithCommute (ArrayRef< int > Mask)
 Helper function that returns true if the shuffle mask should be commuted to improve canonicalization. More...
 
static SDValue lowerVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Top-level lowering for x86 vector shuffles. More...
 
static SDValue lowerVSELECTtoVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a VSELECT instruction to a vector shuffle. More...
 
static SDValue LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerSCALAR_TO_VECTOR (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool areOnlyUsersOf (SDNode *N, ArrayRef< SDValue > ValidUsers)
 
static SDValue LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue GetTLSADDR (SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LocalDynamic=false)
 
static SDValue LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
 
static SDValue LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
 
static SDValue LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool is64Bit)
 
static SDValue LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC)
 
static SDValue LowerShiftParts (SDValue Op, SelectionDAG &DAG)
 Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount. More...
 
static SDValue lowerUINT_TO_FP_v2i32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDLoc &DL)
 
static SDValue lowerUINT_TO_FP_vXi32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerAVXExtend (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerZERO_EXTEND_AVX512 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerANY_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerZERO_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue truncateVectorCompareWithPACKSS (EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Helper to recursively truncate vector elements in half with PACKSS. More...
 
static SDValue LowerTruncateVecI1 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerFP_EXTEND (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFABSorFNEG (SDValue Op, SelectionDAG &DAG)
 The only differences between FABS and FNEG are the mask and the logic op. More...
 
static SDValue LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFGETSIGN (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerVectorAllZeroTest (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool hasNonFlagsUse (SDValue Op)
 return true if Op has a use that doesn't just read flags. More...
 
static SDValue EmitKTEST (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue getSETCC (X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl, SelectionDAG &DAG)
 Helper for creating a X86ISD::SETCC node. More...
 
static SDValue getBitTestCondition (SDValue Src, SDValue BitNo, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG)
 Create a BT (Bit Test) node - Test bit BitNo in Src and set condition according to equal/not-equal condition code CC. More...
 
static SDValue LowerAndToBT (SDValue And, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG)
 Result of 'and' is compared against zero. Change to a BT node if possible. More...
 
static SDValue LowerTruncateToBT (SDValue Op, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG)
 
static int translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1)
 Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs. More...
 
static SDValue Lower256IntVSETCC (SDValue Op, SelectionDAG &DAG)
 Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back. More...
 
static SDValue LowerBoolVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerIntVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG)
 
static SDValue ChangeVSETULTtoVSETULE (const SDLoc &dl, SDValue Op1, SelectionDAG &DAG)
 Try to turn a VSETULT into a VSETULE by modifying its second operand Op1. More...
 
static SDValue LowerVSETCC (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool isX86LogicalCmp (SDValue Op)
 Return true if opcode is a X86 logical comparison. More...
 
static bool isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG)
 
static SDValue LowerSIGN_EXTEND_AVX512 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerEXTEND_VECTOR_INREG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerSIGN_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerTruncatingStore (SDValue StOp, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerExtended1BitVectorLoad (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerExtendedLoad (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool isAndOrOfSetCCs (SDValue Op, unsigned &Opc)
 Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR. More...
 
static bool isXor1OfSetCC (SDValue Op)
 Return true if node is an ISD::XOR of a X86ISD::SETCC and 1 and that the SETCC node has a single use. More...
 
static SDValue LowerVACOPY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue getTargetVShiftByConstNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG)
 Handle vector element shifts where the shift amount is a constant. More...
 
static SDValue getTargetVShiftNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, SDValue ShAmt, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle vector element shifts where the shift amount may or may not be a constant. More...
 
static SDValue getVectorMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics. More...
 
static SDValue getScalarMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Creates an SDNode for a predicated scalar operation. More...
 
static int getSEHRegistrationNodeSize (const Function *Fn)
 
static SDValue recoverFramePointer (SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP)
 When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP. More...
 
static SDValue LowerINTRINSIC_WO_CHAIN (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue getGatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static void getExtendedControlRegister (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
 Handles the lowering of builtin intrinsic that return the value of the extended control register. More...
 
static void getReadPerformanceCounter (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
 Handles the lowering of builtin intrinsics that read performance monitor counters (x86_rdpmc). More...
 
static void getReadTimeStampCounter (SDNode *N, const SDLoc &DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
 Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp). More...
 
static SDValue LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue MarkEHRegistrationNode (SDValue Op, SelectionDAG &DAG)
 
static SDValue MarkEHGuard (SDValue Op, SelectionDAG &DAG)
 
static SDValue EmitTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
 Emit Truncating Store with signed or unsigned saturation. More...
 
static SDValue EmitMaskedTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
 Emit Masked Truncating Store with signed or unsigned saturation. More...
 
static SDValue LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerVectorCTLZ_AVX512 (SDValue Op, SelectionDAG &DAG)
 Lower a vector CTLZ using native supported vector CTLZ instruction. More...
 
static SDValue LowerVectorCTLZInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerVectorCTLZ (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTLZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTTZ (SDValue Op, SelectionDAG &DAG)
 
static SDValue Lower256IntArith (SDValue Op, SelectionDAG &DAG)
 Break a 256-bit integer operation into two new 128-bit ones and then concatenate the result back. More...
 
static SDValue Lower512IntArith (SDValue Op, SelectionDAG &DAG)
 Break a 512-bit integer operation into two new 256-bit ones and then concatenate the result back. More...
 
static SDValue LowerADD (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerSUB (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerMINMAX (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerMUL (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMULH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMUL_LOHI (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool SupportedVectorShiftWithImm (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static bool SupportedVectorShiftWithBaseAmnt (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static bool SupportedVectorVarShift (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static SDValue LowerScalarImmediateShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerScalarVariableShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerRotate (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerXALUO (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerATOMIC_FENCE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCMP_SWAP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerBITCAST (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerHorizontalByteSum (SDValue V, MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Compute the horizontal sum of bytes in V for the elements of VT. More...
 
static SDValue LowerVectorCTPOPInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerVectorCTPOPBitmath (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerVectorCTPOP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTPOP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerBITREVERSE_XOP (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerBITREVERSE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerAtomicArithWithLOCK (SDValue N, SelectionDAG &DAG)
 
static SDValue lowerAtomicArith (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower atomic_load_ops into LOCK-prefixed operations. More...
 
static SDValue LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerADDC_ADDE_SUBC_SUBE (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFSINCOS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue ExtendToType (SDValue InOp, MVT NVT, SelectionDAG &DAG, bool FillWithZeroes=false)
 Widen a vector input to a vector of NVT. More...
 
static SDValue LowerMSCATTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMLOAD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMSTORE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMGATHER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static MachineBasicBlockemitXBegin (MachineInstr &MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII)
 Utility function to emit xbegin specifying the start of an RTM region. More...
 
static MachineBasicBlockemitPCMPSTRM (MachineInstr &MI, MachineBasicBlock *BB, const TargetInstrInfo *TII)
 
static MachineBasicBlockemitPCMPSTRI (MachineInstr &MI, MachineBasicBlock *BB, const TargetInstrInfo *TII)
 
static MachineBasicBlockemitWRPKRU (MachineInstr &MI, MachineBasicBlock *BB, const X86Subtarget &Subtarget)
 
static MachineBasicBlockemitRDPKRU (MachineInstr &MI, MachineBasicBlock *BB, const X86Subtarget &Subtarget)
 
static MachineBasicBlockemitMonitor (MachineInstr &MI, MachineBasicBlock *BB, const X86Subtarget &Subtarget, unsigned Opc)
 
static bool checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
 
static bool isCMOVPseudo (MachineInstr &MI)
 
static bool matchUnaryVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, bool FloatDomain, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT)
 
static bool matchUnaryPermuteVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, bool FloatDomain, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
 
static bool matchBinaryVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, bool FloatDomain, SDValue &V1, SDValue &V2, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, bool IsUnary)
 
static bool matchBinaryPermuteVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, bool FloatDomain, SDValue &V1, SDValue &V2, SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
 
static bool combineX86ShuffleChain (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Combine an arbitrary chain of shuffles into a single instruction if possible. More...
 
static bool combineX86ShufflesConstants (const SmallVectorImpl< SDValue > &Ops, ArrayRef< int > Mask, SDValue Root, bool HasVariableMask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool combineX86ShufflesRecursively (ArrayRef< SDValue > SrcOps, int SrcOpIndex, SDValue Root, ArrayRef< int > RootMask, int Depth, bool HasVariableMask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Fully generic combining of x86 shuffle instructions. More...
 
static SmallVector< int, 4 > getPSHUFShuffleMask (SDValue N)
 Get the PSHUF-style mask from PSHUF node. More...
 
static SDValue combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 Search for a combinable shuffle across a chain ending in pshufd. More...
 
static bool combineRedundantHalfShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 Search for a combinable shuffle across a chain ending in pshuflw or pshufhw. More...
 
static SDValue combineTargetShuffle (SDValue N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Try to combine x86 target specific shuffles. More...
 
static bool isAddSub (SDNode *N, const X86Subtarget &Subtarget, SDValue &Opnd0, SDValue &Opnd1)
 Returns true iff the shuffle node N can be replaced with ADDSUB operation. More...
 
static SDValue combineShuffleToAddSubOrFMAddSub (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to combine a shuffle into a target-specific add-sub or mul-add-sub node. More...
 
static SDValue combineShuffleOfConcatUndef (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue XFormVExtractWithShuffleIntoLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 Check if a vector extract from a target-specific shuffle of a load can be folded into a single element load. More...
 
static SDValue combineBitcast (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue matchBinOpReduction (SDNode *Extract, ISD::NodeType BinOp)
 
static bool detectZextAbsDiff (const SDValue &Select, SDValue &Op0, SDValue &Op1)
 
static SDValue createPSADBW (SelectionDAG &DAG, const SDValue &Zext0, const SDValue &Zext1, const SDLoc &DL)
 
static SDValue combineBasicSADPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineExtractVectorElt (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence. More...
 
static SDValue combineVSelectWithAllOnesOrZeros (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation. More...
 
static SDValue combineSelectOfTwoConstants (SDNode *N, SelectionDAG &DAG)
 
static bool combineBitcastForMaskedOp (SDValue OrigOp, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineSelect (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Do target-specific dag combines on SELECT and VSELECT nodes. More...
 
static SDValue combineSetCCAtomicArith (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG)
 Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction. More...
 
static SDValue checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC)
 
static bool checkBoolTestAndOrSetCCCombine (SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd)
 Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS. More...
 
static SDValue combineSetCCEFLAGS (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG)
 Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values. More...
 
static SDValue combineCMov (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]. More...
 
static bool canReduceVMulWidth (SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode)
 
static SDValue reduceVMULWidth (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code. More...
 
static SDValue combineMul (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Optimize a single multiply with constant into two operations in order to implement it with two cheaper instructions, e.g. More...
 
static SDValue combineShiftLeft (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineShiftRightAlgebraic (SDNode *N, SelectionDAG &DAG)
 
static SDValue performShiftToAllZeros (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Returns a vector of 0s if the node in input is a vector logical shift by a constant amount which is known to be bigger than or equal to the vector element size in bits. More...
 
static SDValue combineShift (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorShift (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineCompareEqual (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends. More...
 
static SDValue combineANDXORWithAllOnesIntoANDNP (SDNode *N, SelectionDAG &DAG)
 Try to fold: (and (xor X, -1), Y) -> (andnp X, Y). More...
 
static SDValue WidenMaskArithmetic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue convertIntLogicToFPLogic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If both input operands of a logic op are being cast from floating point types, try to convert this into a floating point logic node to avoid unnecessary moves from SSE to integer registers. More...
 
static SDValue combinePCMPAnd1 (SDNode *N, SelectionDAG &DAG)
 If this is a PCMPEQ or PCMPGT result that is bitwise-anded with 1 (this is the x86 lowering of a SETCC + ZEXT), replace the 'and' with a shift-right to eliminate loading the vector constant mask value. More...
 
static SDValue combineAnd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineLogicBlendIntoPBLENDV (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerX86CmpEqZeroToCtlzSrl (SDValue Op, EVT ExtTy, SelectionDAG &DAG)
 
static SDValue combineOrCmpEqZeroToCtlzSrl (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineIntegerAbs (SDNode *N, SelectionDAG &DAG)
 Generate NEG and CMOV for integer abs. More...
 
static SDValue foldXorTruncShiftIntoCmp (SDNode *N, SelectionDAG &DAG)
 Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1) More...
 
static SDValue foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1. More...
 
static SDValue detectAVGPattern (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 This function detects the AVG pattern between vectors of unsigned i8/i16, which is c = (a + b + 1) / 2, and replace this operation with the efficient X86ISD::AVG instruction. More...
 
static SDValue combineLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static int getOneTrueElt (SDValue V)
 If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element. More...
 
static bool getParamsForOneTrueMaskedElt (MaskedLoadStoreSDNode *MaskedOp, SelectionDAG &DAG, SDValue &Addr, SDValue &Index, unsigned &Alignment)
 Given a masked memory load/store operation, return true if it has one mask bit set. More...
 
static SDValue reduceMaskedLoadToScalarLoad (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert. More...
 
static SDValue combineMaskedLoadConstantMask (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineMaskedLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue reduceMaskedStoreToScalarStore (MaskedStoreSDNode *MS, SelectionDAG &DAG)
 If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store. More...
 
static SDValue combineMaskedStore (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineStore (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool isHorizontalBinOp (SDValue &LHS, SDValue &RHS, bool IsCommutative)
 Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS. More...
 
static SDValue combineFaddFsub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on floating-point adds/subs. More...
 
static SDValue combineTruncatedArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDLoc &DL)
 Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen. More...
 
static SDValue combineVectorTruncationWithPACKUS (SDNode *N, SelectionDAG &DAG, SmallVector< SDValue, 8 > &Regs)
 Truncate a group of v4i32 into v16i8/v8i16 using X86ISD::PACKUS. More...
 
static SDValue combineVectorTruncationWithPACKSS (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG, SmallVector< SDValue, 8 > &Regs)
 Truncate a group of v4i32 into v8i16 using X86ISD::PACKSS. More...
 
static SDValue combineVectorTruncation (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 This function transforms truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations. More...
 
static SDValue combineVectorSignBitsTruncation (SDNode *N, SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 This function transforms vector truncation of 'all or none' bits values. More...
 
static SDValue combineTruncate (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue isFNEG (SDNode *N)
 Returns the negated value if the node N flips sign of FP value. More...
 
static SDValue combineFneg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on floating point negations. More...
 
static SDValue lowerX86FPLogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineXor (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool isNullFPScalarOrVectorConst (SDValue V)
 
static SDValue getNullFPConstForNullVal (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value. More...
 
static SDValue combineFAndFNotToFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineFAnd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FAND nodes. More...
 
static SDValue combineFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FANDN nodes. More...
 
static SDValue combineFOr (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes. More...
 
static SDValue combineFMinFMax (SDNode *N, SelectionDAG &DAG)
 Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes. More...
 
static SDValue combineFMinNumFMaxNum (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineSignExtendInReg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue promoteExtBeforeAdd (SDNode *Ext, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 sext(add_nsw(x, C)) –> add(sext(x), C_sext) zext(add_nuw(x, C)) –> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode. More...
 
static SDValue getDivRem8 (SDNode *N, SelectionDAG &DAG)
 (i8,i32 {s/z}ext ({s/u}divrem (i8 x, i8 y)) -> (i8,i32 ({s/u}divrem_sext_hreg (i8 x, i8 y) This exposes the {s/z}ext to the sdivrem lowering, so that it directly extends from AH (which we otherwise need to do contortions to access). More...
 
static SDValue combineToExtendVectorInReg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Convert a SEXT or ZEXT of a vector to a SIGN_EXTEND_VECTOR_INREG or ZERO_EXTEND_VECTOR_INREG, this requires the splitting (or concatenating with UNDEFs) of the input to vectors of the same size as the target type which then extends the lowest elements. More...
 
static SDValue combineSext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineFMA (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineZext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineSetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Optimize x == -y –> x+y == 0 x != -y –> x+y != 0. More...
 
static SDValue combineGatherScatter (SDNode *N, SelectionDAG &DAG)
 
static SDValue MaterializeSETB (const SDLoc &DL, SDValue EFLAGS, SelectionDAG &DAG, MVT VT)
 
static SDValue combineX86SetCC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineBrCond (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Optimize branch condition evaluation. More...
 
static SDValue combineVectorCompareAndMaskUnaryOp (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineUIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineADC (SDNode *N, SelectionDAG &DAG, X86TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue OptimizeConditionalInDecrement (SDNode *N, SelectionDAG &DAG)
 fold (add Y, (sete X, 0)) -> adc 0, Y (add Y, (setne X, 0)) -> sbb -1, Y (sub (sete X, 0), Y) -> sbb 0, Y (sub (setne X, 0), Y) -> adc -1, Y More...
 
static SDValue combineLoopSADPattern (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineAdd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineVSZext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineLockSub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Canonicalize (LSUB p, 1) -> (LADD p, -1). More...
 
static SDValue combineTestM (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineVectorCompare (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool matchAsm (StringRef S, ArrayRef< const char * > Pieces)
 
static bool clobbersFlagRegisters (const SmallVector< StringRef, 4 > &AsmPieces)
 
static bool isGRClass (const TargetRegisterClass &RC)
 Check if RC is a general purpose register class. More...
 
static bool isFRClass (const TargetRegisterClass &RC)
 Check if RC is a vector register class. More...
 

Variables

static cl::opt< boolExperimentalVectorWideningLegalization ("x86-experimental-vector-widening-legalization", cl::init(false), cl::desc("Enable an experimental vector type legalization through widening ""rather than promotion."), cl::Hidden)
 

Macro Definition Documentation

#define DEBUG_TYPE   "x86-isel"

Definition at line 63 of file X86ISelLowering.cpp.

Enumeration Type Documentation

enum ShrinkMode

Different mul shrinking modes.

Enumerator
MULS8 
MULU8 
MULS16 
MULU16 

Definition at line 29931 of file X86ISelLowering.cpp.

CallIsStructReturn - Determines whether a call uses struct return semantics.

Enumerator
NotStructReturn 
RegStructReturn 
StackStructReturn 

Definition at line 2569 of file X86ISelLowering.cpp.

Function Documentation

static bool areOnlyUsersOf ( SDNode N,
ArrayRef< SDValue ValidUsers 
)
static
static StructReturnType argsAreStructReturn ( const SmallVectorImpl< ISD::InputArg > &  Ins,
bool  IsMCU 
)
static

Determines whether a function uses struct return semantics.

Definition at line 2589 of file X86ISelLowering.cpp.

References llvm::SmallVectorBase::empty(), fuzzer::Flags, llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSRet(), NotStructReturn, RegStructReturn, and StackStructReturn.

static SDValue buildFromShuffleMostly ( SDValue  Op,
SelectionDAG DAG 
)
static
static StructReturnType callIsStructReturn ( const SmallVectorImpl< ISD::OutputArg > &  Outs,
bool  IsMCU 
)
static
static bool canGuaranteeTCO ( CallingConv::ID  CC)
static

Return true if the calling convention is one that we can guarantee TCO for.

Definition at line 2616 of file X86ISelLowering.cpp.

References llvm::CallingConv::Fast, llvm::CallingConv::GHC, llvm::CallingConv::HHVM, llvm::CallingConv::HiPE, and llvm::CallingConv::X86_RegCall.

Referenced by mayTailCallThisCC(), and shouldGuaranteeTCO().

static int canLowerByDroppingEvenElements ( ArrayRef< int >  Mask,
bool  IsSingleInput 
)
static

Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped.

This handles shuffles which take every Nth element where N is a power of two. Example shuffle masks:

N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24

Any of these lanes can of course be undef.

This routine only supports N <= 3. FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here for larger N.

Returns
N above, or the number of times even elements must be dropped if there is such a number. Otherwise returns zero.

Definition at line 11025 of file X86ISelLowering.cpp.

References llvm::array_lengthof(), assert(), i, llvm::isPowerOf2_32(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I8VectorShuffle().

static bool canonicalizeShuffleMaskWithCommute ( ArrayRef< int >  Mask)
static

Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.

Definition at line 13279 of file X86ISelLowering.cpp.

References assert(), i, and llvm::ArrayRef< T >::size().

Referenced by combineX86ShufflesRecursively(), and lowerVectorShuffle().

static bool canReduceVMulWidth ( SDNode N,
SelectionDAG DAG,
ShrinkMode Mode 
)
static
static bool canWidenShuffleElements ( ArrayRef< int >  Mask,
SmallVectorImpl< int > &  WidenedMask 
)
static

Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.

Appends the mask for wider elements in WidenedMask if valid. Otherwise leaves it in an unspecified state.

NOTE: This must handle normal vector shuffle masks and target vector shuffle masks. The latter have the special property of a '-2' representing a zero-ed lane of a vector.

Definition at line 4554 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), i, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineX86ShufflesRecursively(), lowerV2X128VectorShuffle(), lowerV4X128VectorShuffle(), and lowerVectorShuffle().

static SDValue ChangeVSETULTtoVSETULE ( const SDLoc dl,
SDValue  Op1,
SelectionDAG DAG 
)
static
static bool checkAndUpdateEFLAGSKill ( MachineBasicBlock::iterator  SelectItr,
MachineBasicBlock BB,
const TargetRegisterInfo TRI 
)
static
static bool checkBoolTestAndOrSetCCCombine ( SDValue  Cond,
X86::CondCode CC0,
X86::CondCode CC1,
SDValue Flags,
bool isAnd 
)
static

Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.

Match: (X86or (X86setcc) (X86setcc)) (X86cmp (and (X86setcc) (X86setcc)), 0)

Definition at line 29676 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::AND, llvm::X86ISD::CMP, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::isNullConstant(), LLVM_FALLTHROUGH, llvm::ISD::OR, llvm::X86ISD::OR, and llvm::X86ISD::SETCC.

Referenced by combineCMov().

static SDValue checkBoolTestSetCCCombine ( SDValue  Cmp,
X86::CondCode CC 
)
static
static bool clobbersFlagRegisters ( const SmallVector< StringRef, 4 > &  AsmPieces)
static
static SDValue combineADC ( SDNode N,
SelectionDAG DAG,
X86TargetLowering::DAGCombinerInfo &  DCI 
)
static
static SDValue combineAdd ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineAnd ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineANDXORWithAllOnesIntoANDNP ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue combineBasicSADPattern ( SDNode Extract,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineBitcast ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static bool combineBitcastForMaskedOp ( SDValue  OrigOp,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static
static SDValue combineBrCond ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineBT ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static
static SDValue combineCMov ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineCompareEqual ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineExtractVectorElt ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.

For i686, the best sequence is apparently storing the value and loading scalars back, while for x64 we should use 64-bit extracts and shifts.

Definition at line 28650 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::SmallVectorTemplateCommon< T >::begin(), llvm::ISD::BITCAST, combineBasicSADPattern(), llvm::SelectionDAG::CreateStackTemporary(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SDValue::getResNo(), llvm::TargetLoweringBase::getShiftAmountTy(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::TargetLoweringBase::getVectorIdxTy(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), i, llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::TargetLoweringBase::isOperationLegal(), llvm::X86ISD::MMX_MOVD2W, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::x86mmx, XFormVExtractWithShuffleIntoLoad(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue combineFaddFsub ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineFAnd ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on X86ISD::FAND nodes.

Definition at line 32680 of file X86ISelLowering.cpp.

References combineFAndFNotToFAndn(), getNullFPConstForNullVal(), llvm::SDNode::getOperand(), and lowerX86FPLogicOp().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue combineFAndFNotToFAndn ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineFAndn ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on X86ISD::FANDN nodes.

Definition at line 32697 of file X86ISelLowering.cpp.

References getNullFPConstForNullVal(), llvm::SDNode::getOperand(), isNullFPScalarOrVectorConst(), and lowerX86FPLogicOp().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue combineFMA ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineFMinFMax ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue combineFMinNumFMaxNum ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineFneg ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineFOr ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineGatherScatter ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue combineIntegerAbs ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue combineLoad ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineLockSub ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineLogicBlendIntoPBLENDV ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineLoopSADPattern ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineMaskedLoad ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineMaskedLoadConstantMask ( MaskedLoadSDNode ML,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static
static SDValue combineMaskedStore ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineMul ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineOr ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineOrCmpEqZeroToCtlzSrl ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combinePCMPAnd1 ( SDNode N,
SelectionDAG DAG 
)
static

If this is a PCMPEQ or PCMPGT result that is bitwise-anded with 1 (this is the x86 lowering of a SETCC + ZEXT), replace the 'and' with a shift-right to eliminate loading the vector constant mask value.

This relies on the fact that a PCMP always creates an all-ones or all-zeros bitmask per element.

Definition at line 30703 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i8, llvm::ISD::isConstantSplatVector(), llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, peekThroughBitcasts(), and llvm::X86ISD::VSRLI.

Referenced by combineAnd().

static SDValue combineRedundantDWordShuffle ( SDValue  N,
MutableArrayRef< int >  Mask,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static
static bool combineRedundantHalfShuffle ( SDValue  N,
MutableArrayRef< int >  Mask,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

Search for a combinable shuffle across a chain ending in pshuflw or pshufhw.

We walk up the chain, skipping shuffles of the other half and looking through shuffles which switch halves trying to find a shuffle of the same pair of dwords.

Definition at line 27697 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPSHUFShuffleMask(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::hasOneUse(), llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, and llvm::MVT::v8i16.

Referenced by combineTargetShuffle().

static SDValue combineSelect ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on SELECT and VSELECT nodes.

Definition at line 29096 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), assert(), combineBitcastForMaskedOp(), combineSelectOfTwoConstants(), combineVSelectWithAllOnesOrZeros(), llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), E, llvm::MVT::f128, llvm::MVT::f32, llvm::MVT::f80, llvm::X86ISD::FMAX, llvm::X86ISD::FMIN, llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasVLX(), I, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i8, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SelectionDAG::isEqualTo(), llvm::EVT::isFloatingPoint(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::SelectionDAG::isKnownNeverZero(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::TargetMachine::Options, Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::X86ISD::SHRUNKBLEND, llvm::ISD::SIGN_EXTEND, llvm::TargetLowering::SimplifyDemandedBits(), llvm::ISD::SUB, llvm::X86ISD::SUBUS, std::swap(), llvm::TargetOptions::UnsafeFPMath, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v2f32, llvm::MVT::v32i8, llvm::MVT::v8i16, llvm::ISD::VSELECT, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue combineSelectOfTwoConstants ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue combineSetCC ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineSetCCAtomicArith ( SDValue  Cmp,
X86::CondCode CC,
SelectionDAG DAG 
)
static
static SDValue combineSetCCEFLAGS ( SDValue  EFLAGS,
X86::CondCode CC,
SelectionDAG DAG 
)
static

Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values.

Definition at line 29717 of file X86ISelLowering.cpp.

References checkBoolTestSetCCCombine(), and combineSetCCAtomicArith().

Referenced by combineBrCond(), combineCMov(), and combineX86SetCC().

static SDValue combineSext ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineShift ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineShiftLeft ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue combineShiftRightAlgebraic ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue combineShuffle ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineShuffleOfConcatUndef ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineShuffleToAddSubOrFMAddSub ( SDNode N,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to combine a shuffle into a target-specific add-sub or mul-add-sub node.

Definition at line 28102 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADDSUB, llvm::X86ISD::FMADDSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getValueType(), llvm::EVT::is512BitVector(), isAddSub(), and isFMAddSub().

Referenced by combineShuffle().

static SDValue combineSignExtendInReg ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineSIntToFP ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineStore ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Definition at line 31791 of file X86ISelLowering.cpp.

References llvm::TargetLoweringBase::allowsMemoryAccess(), assert(), detectAVGPattern(), extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::CallingConv::Fast, llvm::MemSDNode::getAddressSpace(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::MVT::getStoreSize(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MachinePointerInfo::getWithOffset(), llvm::Function::hasFnAttribute(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), i, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::MVT::integer_valuetypes(), llvm::X86Subtarget::is64Bit(), llvm::ISD::isNormalLoad(), llvm::isPowerOf2_32(), llvm::StoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegalOrCustom(), llvm::TargetLoweringBase::isTypeLegal(), isVolatile(), llvm::MemSDNode::isVolatile(), fuzzer::min(), llvm::MinAlign(), N, llvm::MVT::Other, Ptr, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ISD::TokenFactor, and llvm::X86Subtarget::useSoftFloat().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue combineSub ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineTargetShuffle ( SDValue  N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Try to combine x86 target specific shuffles.

Definition at line 27760 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::X86ISD::BLENDI, combineRedundantDWordShuffle(), combineRedundantHalfShuffle(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), D, llvm::X86ISD::FHADD, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), getPSHUFShuffleMask(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), i, llvm::MVT::i32, llvm::MVT::i8, llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::HexagonMCInstrInfo::isFloat(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), isShuffleEquivalent(), isShuffleFoldableLoad(), llvm::SDValue::isUndef(), isUndefOrZero(), llvm::makeArrayRef(), llvm::BitmaskEnumDetail::Mask(), llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, peekThroughBitcasts(), peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, setTargetShuffleZeroElements(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4i32, and llvm::ISD::VECTOR_SHUFFLE.

Referenced by combineShuffle().

static SDValue combineTestM ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue combineToExtendVectorInReg ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineTruncate ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineTruncatedArithmetic ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget,
SDLoc DL 
)
static
static SDValue combineUIntToFP ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineVectorCompare ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineVectorCompareAndMaskUnaryOp ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue combineVectorShift ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineVectorSignBitsTruncation ( SDNode N,
SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue combineVectorTruncation ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

This function transforms truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations.

We do it here because after type legalization the truncation will be translated into a BUILD_VECTOR with each element that is extracted from a vector and then truncated, and it is difficult to do this optimization based on them.

Definition at line 32353 of file X86ISelLowering.cpp.

References combineVectorTruncationWithPACKSS(), combineVectorTruncationWithPACKUS(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::isPowerOf2_32(), llvm::EVT::isSimple(), and llvm::EVT::isVector().

Referenced by combineTruncate().

static SDValue combineVectorTruncationWithPACKSS ( SDNode N,
const X86Subtarget Subtarget,
SelectionDAG DAG,
SmallVector< SDValue, 8 > &  Regs 
)
static
static SDValue combineVectorTruncationWithPACKUS ( SDNode N,
SelectionDAG DAG,
SmallVector< SDValue, 8 > &  Regs 
)
static
static SDValue combineVSelectWithAllOnesOrZeros ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineVSZext ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineX86SetCC ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static bool combineX86ShuffleChain ( ArrayRef< SDValue Inputs,
SDValue  Root,
ArrayRef< int >  BaseMask,
int  Depth,
bool  HasVariableMask,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Combine an arbitrary chain of shuffles into a single instruction if possible.

This is the leaf of the recursive combine below. When we have found some chain of single-use x86 shuffle instructions and accumulated the combined shuffle mask represented by them, this will try to pattern match that mask into either a single instruction if there is a special purpose instruction for this operation, or into a PSHUFB instruction which is a fully general instruction but should only be used to replace chains over a certain depth.

Definition at line 26754 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::AND, llvm::any_of(), assert(), llvm::ArrayRef< T >::begin(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SmallVectorTemplateCommon< T >::data(), llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::end(), llvm::X86ISD::FAND, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::APInt::getNullValue(), llvm::SDValue::getOpcode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSSE3(), llvm::X86Subtarget::hasVBMI(), llvm::X86Subtarget::hasVLX(), llvm::X86Subtarget::hasXOP(), i, llvm::MVT::i32, llvm::MVT::i8, is128BitLaneCrossingShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), isSequentialOrUndefInRange(), isSequentialOrUndefOrZeroInRange(), llvm::TargetLoweringBase::isTypeLegal(), isUndefOrZeroOrInRange(), llvm::BitmaskEnumDetail::Mask(), matchBinaryPermuteVectorShuffle(), matchBinaryVectorShuffle(), matchUnaryPermuteVectorShuffle(), matchUnaryVectorShuffle(), peekThroughBitcasts(), llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), scaleShuffleMask(), llvm::SmallVectorTemplateCommon< T >::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::MVT::v16f32, llvm::MVT::v16i16, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, llvm::MVT::v32i16, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i64, llvm::MVT::v64i8, llvm::MVT::v8f32, llvm::MVT::v8f64, llvm::MVT::v8i32, llvm::MVT::v8i64, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, and llvm::X86ISD::VZEXT_LOAD.

Referenced by combineX86ShufflesRecursively().

static bool combineX86ShufflesConstants ( const SmallVectorImpl< SDValue > &  Ops,
ArrayRef< int >  Mask,
SDValue  Root,
bool  HasVariableMask,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static bool combineX86ShufflesRecursively ( ArrayRef< SDValue SrcOps,
int  SrcOpIndex,
SDValue  Root,
ArrayRef< int >  RootMask,
int  Depth,
bool  HasVariableMask,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Fully generic combining of x86 shuffle instructions.

This should be the last combine run over the x86 shuffle instructions. Once they have been fully optimized, this will recursively consider all chains of single-use shuffle instructions, build a generic model of the cumulative shuffle operation, and check for simpler instructions which implement this operation. We use this primarily for two purposes:

1) Collapse generic shuffles to specialized single instructions when equivalent. In most cases, this is just an encoding size win, but sometimes we will collapse multiple generic shuffles into a single special-purpose shuffle. 2) Look for sequences of shuffle instructions with 3 or more total instructions, and replace them with the slightly more expensive SSSE3 PSHUFB instruction if available. We do this as the last combining step to ensure we avoid using PSHUFB if we can implement the shuffle with a suitable short sequence of other instructions. The PSHUFB will either use a register or have to read from memory and so is slightly (but only slightly) more expensive than the other shuffle instructions.

Because this is inherently a quadratic operation (for each shuffle in a chain, we recurse up the chain), the depth is limited to 8 instructions. This should never be an issue in practice as the shuffle lowering doesn't produce sequences of more than 8 instructions.

FIXME: We will currently miss some cases where the redundant shuffling would simplify under the threshold for PSHUFB formation because of combine-ordering. To fix this, we should do the redundant instruction combining in this recursive walk.

Definition at line 27330 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::any_of(), assert(), llvm::ArrayRef< T >::begin(), canonicalizeShuffleMaskWithCommute(), canWidenShuffleElements(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineX86ShuffleChain(), combineX86ShufflesConstants(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::SmallVectorBase::empty(), llvm::ArrayRef< T >::end(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), getZeroVector(), i, llvm::SDNode::isOnlyUserOf(), isTargetShuffleVariableMask(), llvm::MVT::isVector(), llvm::BitmaskEnumDetail::Mask(), peekThroughBitcasts(), peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SmallVectorImpl< T >::reserve(), resolveTargetShuffleInputs(), llvm::SmallVectorTemplateCommon< T >::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and std::swap().

Referenced by combineAnd(), combineShuffle(), and combineVectorShift().

static SDValue combineXor ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineZext ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SmallBitVector computeZeroableShuffleElements ( ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2 
)
static

Compute whether each element of a shuffle is zeroable.

A "zeroable" vector shuffle element is one which can be lowered to zero. Either it is an undef element in the shuffle mask, the element of the input referenced is undef, or the element of the input referenced is known to be zero. Many x86 shuffles can zero lanes cheaply and we often want to handle as many lanes with this technique as possible to simplify the remaining shuffle.

Definition at line 8025 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::APInt::getLoBits(), llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), i, llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), llvm::APInt::lshr(), peekThroughBitcasts(), llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.

Referenced by lowerVectorShuffle().

static SDValue concat128BitVectors ( SDValue  V1,
SDValue  V2,
EVT  VT,
unsigned  NumElems,
SelectionDAG DAG,
const SDLoc dl 
)
static

Concat two 128-bit vectors into a 256 bit vector using VINSERTF128 instructions.

This is used because creating CONCAT_VECTOR nodes of BUILD_VECTORS returns a larger BUILD_VECTOR while we're trying to lower large BUILD_VECTORS.

Definition at line 5082 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getUNDEF(), and insert128BitVector().

Referenced by getOnesVector(), and LowerAVXCONCAT_VECTORS().

static SDValue concat256BitVectors ( SDValue  V1,
SDValue  V2,
EVT  VT,
unsigned  NumElems,
SelectionDAG DAG,
const SDLoc dl 
)
static

Definition at line 5089 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getUNDEF(), and insert256BitVector().

Referenced by LowerAVXCONCAT_VECTORS().

static unsigned convertBitVectorToUnsiged ( const SmallBitVector Zeroable)
static

Definition at line 8185 of file X86ISelLowering.cpp.

References i, and llvm::SmallBitVector::size().

Referenced by lowerVectorShuffleToEXPAND().

static SDValue ConvertI1VectorToInteger ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue convertIntLogicToFPLogic ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue CreateCopyOfByValArgument ( SDValue  Src,
SDValue  Dst,
SDValue  Chain,
ISD::ArgFlagsTy  Flags,
SelectionDAG DAG,
const SDLoc dl 
)
static

Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute.

The copy will be passed as a byval function parameter.

Definition at line 2604 of file X86ISelLowering.cpp.

References llvm::ISD::ArgFlagsTy::getByValAlign(), llvm::ISD::ArgFlagsTy::getByValSize(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMemcpy(), and llvm::MVT::i32.

static SDValue createPSADBW ( SelectionDAG DAG,
const SDValue Zext0,
const SDValue Zext1,
const SDLoc DL 
)
static
static void createUnpackShuffleMask ( MVT  VT,
SmallVectorImpl< int > &  Mask,
bool  Lo,
bool  Unary 
)
static
static SDValue detectAVGPattern ( SDValue  In,
EVT  VT,
SelectionDAG DAG,
const X86Subtarget Subtarget,
const SDLoc DL 
)
static
static bool detectZextAbsDiff ( const SDValue Select,
SDValue Op0,
SDValue Op1 
)
static
static SDValue EltsFromConsecutiveLoads ( EVT  VT,
ArrayRef< SDValue Elts,
SDLoc DL,
SelectionDAG DAG,
bool  isAfterLegalize 
)
static

Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.

Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a

Definition at line 6214 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), assert(), llvm::SmallBitVector::count(), llvm::count(), llvm::SmallBitVector::find_first(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MachineMemOperand::getFlags(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::EVT::getStoreSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasAnyUseOfValue(), i, llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::ISD::isNON_EXTLoad(), llvm::TargetLoweringBase::isOperationLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), llvm::ARM_MB::LD, llvm::ISD::LOAD, llvm::MachineMemOperand::MOVolatile, llvm::MVT::Other, peekThroughBitcasts(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ArrayRef< T >::size(), llvm::ISD::TokenFactor, llvm::SelectionDAG::UpdateNodeOperands(), and llvm::X86ISD::VZEXT_LOAD.

Referenced by combineShuffle(), and LowerINSERT_SUBVECTOR().

static SDValue EmitKTEST ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue EmitMaskedTruncSStore ( bool  SignedSat,
SDValue  Chain,
const SDLoc Dl,
SDValue  Val,
SDValue  Ptr,
SDValue  Mask,
EVT  MemVT,
MachineMemOperand MMO,
SelectionDAG DAG 
)
static

Emit Masked Truncating Store with signed or unsigned saturation.

Definition at line 19872 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getTargetMemSDNode(), llvm::SelectionDAG::getVTList(), llvm::BitmaskEnumDetail::Mask(), llvm::MVT::Other, and Ptr.

Referenced by LowerINTRINSIC_W_CHAIN().

static MachineBasicBlock* emitMonitor ( MachineInstr MI,
MachineBasicBlock BB,
const X86Subtarget Subtarget,
unsigned  Opc 
)
static
static MachineBasicBlock* emitPCMPSTRI ( MachineInstr MI,
MachineBasicBlock BB,
const TargetInstrInfo TII 
)
static
static MachineBasicBlock* emitPCMPSTRM ( MachineInstr MI,
MachineBasicBlock BB,
const TargetInstrInfo TII 
)
static
static MachineBasicBlock* emitRDPKRU ( MachineInstr MI,
MachineBasicBlock BB,
const X86Subtarget Subtarget 
)
static
static SDValue EmitTailCallStoreRetAddr ( SelectionDAG DAG,
MachineFunction MF,
SDValue  Chain,
SDValue  RetAddrFrIdx,
EVT  PtrVT,
unsigned  SlotSize,
int  FPDiff,
const SDLoc dl 
)
static
static SDValue EmitTruncSStore ( bool  SignedSat,
SDValue  Chain,
const SDLoc Dl,
SDValue  Val,
SDValue  Ptr,
EVT  MemVT,
MachineMemOperand MMO,
SelectionDAG DAG 
)
static

Emit Truncating Store with signed or unsigned saturation.

Definition at line 19858 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getTargetMemSDNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::Other, and Ptr.

Referenced by LowerINTRINSIC_W_CHAIN().

static MachineBasicBlock* emitWRPKRU ( MachineInstr MI,
MachineBasicBlock BB,
const X86Subtarget Subtarget 
)
static
static MachineBasicBlock* emitXBegin ( MachineInstr MI,
MachineBasicBlock MBB,
const TargetInstrInfo TII 
)
static
static SDValue ExpandHorizontalBinOp ( const SDValue V0,
const SDValue V1,
const SDLoc DL,
SelectionDAG DAG,
unsigned  X86Opcode,
bool  Mode,
bool  isUndefLO,
bool  isUndefHI 
)
static

Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.

This is a helper function of LowerToHorizontalOp(). This function expects two 256-bit vectors called V0 and V1. At first, each vector is split into two separate 128-bit vectors. Then, the resulting 128-bit vectors are used to implement two horizontal binary operations.

The kind of horizontal binary operation is defined by X86Opcode.

Mode specifies how the 128-bit parts of V0 and V1 are passed in input to the two new horizontal binop. When Mode is set, the first horizontal binop dag node would take as input the lower 128-bit of V0 and the upper 128-bit of V0. The second horizontal binop dag node would take as input the lower 128-bit of V1 and the upper 128-bit of V1. Example: HADD V0_LO, V0_HI HADD V1_LO, V1_HI

Otherwise, the first horizontal binop dag node takes as input the lower 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop dag node takes the upper 128-bit of V0 and the upper 128-bit of V1. Example: HADD V0_LO, V1_LO HADD V0_HI, V1_HI

If isUndefLO is set, then the algorithm propagates UNDEF to the lower 128-bits of the result. If isUndefHI is set, then UNDEF is propagated to the upper 128-bits of the result.

Definition at line 6928 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::AArch64CC::HI, llvm::MVT::is256BitVector(), llvm::SDNode::isUndef(), and llvm::AArch64CC::LO.

Referenced by LowerToHorizontalOp().

static SDValue ExtendToType ( SDValue  InOp,
MVT  NVT,
SelectionDAG DAG,
bool  FillWithZeroes = false 
)
static
static SDValue extract128BitVector ( SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
const SDLoc dl 
)
static

Generate a DAG to grab 128-bits from a vector > 128 bits.

This sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128 or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4 instructions or a simple subregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering EXTRACT_VECTOR_ELT operations easier.

Definition at line 4874 of file X86ISelLowering.cpp.

References assert(), extractSubVector(), llvm::SDValue::getValueType(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().

Referenced by combineStore(), ExpandHorizontalBinOp(), Lower256IntArith(), Lower256IntVSETCC(), LowerBITREVERSE(), LowerBITREVERSE_XOP(), LowerEXTRACT_SUBVECTOR(), LowerMUL_LOHI(), LowerVectorCTLZ(), and LowerVectorCTPOP().

static SDValue extract256BitVector ( SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
const SDLoc dl 
)
static

Generate a DAG to grab 256-bits from a 512-bit vector.

Definition at line 4882 of file X86ISelLowering.cpp.

References assert(), extractSubVector(), llvm::SDValue::getValueType(), and llvm::EVT::is512BitVector().

Referenced by Lower512IntArith(), LowerEXTRACT_SUBVECTOR(), and LowerVectorCTPOP().

static SDValue extractSubVector ( SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
const SDLoc dl,
unsigned  vectorWidth 
)
static
static SDValue foldVectorXorShiftIntoCmp ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue foldXorTruncShiftIntoCmp ( SDNode N,
SelectionDAG DAG 
)
static
static ArrayRef<MCPhysReg> get64BitArgumentGPRs ( CallingConv::ID  CallConv,
const X86Subtarget Subtarget 
)
static
static ArrayRef<MCPhysReg> get64BitArgumentXMMs ( MachineFunction MF,
CallingConv::ID  CallConv,
const X86Subtarget Subtarget 
)
static
static SDValue getBitTestCondition ( SDValue  Src,
SDValue  BitNo,
ISD::CondCode  CC,
const SDLoc dl,
SelectionDAG DAG 
)
static
static Constant* getConstantVector ( MVT  VT,
APInt  SplatValue,
unsigned  SplatBitSize,
LLVMContext C 
)
static
static SDValue getConstVector ( ArrayRef< int >  Values,
MVT  VT,
SelectionDAG DAG,
const SDLoc dl,
bool  IsMask = false 
)
static
static SDValue getConstVector ( ArrayRef< APInt Bits,
SmallBitVector Undefs,
MVT  VT,
SelectionDAG DAG,
const SDLoc dl 
)
static
static SDValue getDivRem8 ( SDNode N,
SelectionDAG DAG 
)
static
static void getExtendedControlRegister ( SDNode N,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget,
SmallVectorImpl< SDValue > &  Results 
)
static
static unsigned getExtractVEXTRACTImmediate ( SDNode N,
unsigned  vecWidth 
)
static
static bool getFauxShuffleMask ( SDValue  N,
SmallVectorImpl< int > &  Mask,
SmallVectorImpl< SDValue > &  Ops 
)
static
static SDValue getGatherNode ( unsigned  Opc,
SDValue  Op,
SelectionDAG DAG,
SDValue  Src,
SDValue  Mask,
SDValue  Base,
SDValue  Index,
SDValue  ScaleOp,
SDValue  Chain,
const X86Subtarget Subtarget 
)
static
static unsigned getInsertVINSERTImmediate ( SDNode N,
unsigned  vecWidth 
)
static
static SDValue getMaskNode ( SDValue  Mask,
MVT  MaskVT,
const X86Subtarget Subtarget,
SelectionDAG DAG,
const SDLoc dl 
)
static
static void getMaxByValAlign ( Type Ty,
unsigned MaxAlign 
)
static

Helper for getByValTypeAlignment to determine the desired ByVal argument alignment.

Definition at line 1788 of file X86ISelLowering.cpp.

Referenced by llvm::X86TargetLowering::getByValTypeAlignment().

static SDValue getMOVL ( SelectionDAG DAG,
const SDLoc dl,
MVT  VT,
SDValue  V1,
SDValue  V2 
)
static

Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width.

Definition at line 3204 of file X86ISelLowering.cpp.

References llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), i, llvm::BitmaskEnumDetail::Mask(), and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().

static SDValue getNullFPConstForNullVal ( SDValue  V,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value.

In the case of a vector, the returned constant will not contain undefined elements even if the input parameter does. This makes it suitable to be used as a replacement operand with operations (eg, bitwise-and) where an undef should not propagate.

Definition at line 32640 of file X86ISelLowering.cpp.

References llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), getZeroVector(), isNullFPScalarOrVectorConst(), and llvm::EVT::isVector().

Referenced by combineFAnd(), and combineFAndn().

static SDValue getOnesVector ( EVT  VT,
const X86Subtarget Subtarget,
SelectionDAG DAG,
const SDLoc dl 
)
static

Returns a vector of specified type with all bits set.

Always build ones vectors as <4 x i32> or <8 x i32>. For 256-bit types with no AVX2 support, use two <4 x i32> inserted in a <8 x i32> appropriately. Then bitcast to their original type, ensuring they get CSE'd.

Definition at line 5100 of file X86ISelLowering.cpp.

References assert(), concat128BitVectors(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::MVT::i32, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::MVT::v4i32, and llvm::MVT::v8i32.

Referenced by combineVectorCompare(), lower1BitVectorShuffle(), LowerSIGN_EXTEND_AVX512(), and materializeVectorConstant().

static int getOneTrueElt ( SDValue  V)
static

If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element.

Otherwise, return -1.

Definition at line 31447 of file X86ISelLowering.cpp.

References llvm::dyn_cast(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), i, llvm::MVT::i1, and llvm::SDValue::isUndef().

Referenced by getParamsForOneTrueMaskedElt().

static bool getParamsForOneTrueMaskedElt ( MaskedLoadStoreSDNode MaskedOp,
SelectionDAG DAG,
SDValue Addr,
SDValue Index,
unsigned Alignment 
)
static

Given a masked memory load/store operation, return true if it has one mask bit set.

If it has one mask bit set, then also return the memory address of the scalar element to load/store, the vector index to insert/extract that scalar element, and the alignment for the scalar memory access.

Definition at line 31482 of file X86ISelLowering.cpp.

References llvm::MemSDNode::getAlignment(), llvm::MaskedLoadStoreSDNode::getBasePtr(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemoryVT(), getOneTrueElt(), llvm::EVT::getStoreSize(), llvm::EVT::getVectorElementType(), and llvm::MinAlign().

Referenced by reduceMaskedLoadToScalarLoad(), and reduceMaskedStoreToScalarStore().

static SDValue getPrefetchNode ( unsigned  Opc,
SDValue  Op,
SelectionDAG DAG,
SDValue  Mask,
SDValue  Base,
SDValue  Index,
SDValue  ScaleOp,
SDValue  Chain,
const X86Subtarget Subtarget 
)
static
static SmallVector<int, 4> getPSHUFShuffleMask ( SDValue  N)
static

Get the PSHUF-style mask from PSHUF node.

This is a very minor wrapper around getTargetShuffleMask to easy forming v4 PSHUF-style masks that can be reused with such instructions.

Definition at line 27518 of file X86ISelLowering.cpp.

References assert(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetShuffleMask(), i, llvm_unreachable, llvm::BitmaskEnumDetail::Mask(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, and llvm::X86ISD::PSHUFLW.

Referenced by combineRedundantDWordShuffle(), combineRedundantHalfShuffle(), and combineTargetShuffle().

static void getReadPerformanceCounter ( SDNode N,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget,
SmallVectorImpl< SDValue > &  Results 
)
static
static void getReadTimeStampCounter ( SDNode N,
const SDLoc DL,
unsigned  Opcode,
SelectionDAG DAG,
const X86Subtarget Subtarget,
SmallVectorImpl< SDValue > &  Results 
)
static
static SDValue getScalarMaskingNode ( SDValue  Op,
SDValue  Mask,
SDValue  PreservedSrc,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Creates an SDNode for a predicated scalar operation.

Returns
(X86vselect Mask, Op, PreservedSrc). The mask is coming as MVT::i8 and it should be truncated to MVT::i1 while lowering masking intrinsics. The main difference between ScalarMaskingNode and VectorMaskingNode is using "X86select" instead of "vselect". We just can't create the "vselect" node for a scalar instruction.

Definition at line 18700 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::FSETCCM, llvm::X86ISD::FSETCCM_RND, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getSimpleValueType(), getZeroVector(), llvm::MVT::i1, llvm::isAllOnesConstant(), llvm::SDValue::isUndef(), llvm::ISD::OR, llvm::X86ISD::SELECTS, llvm::ISD::TRUNCATE, llvm::X86ISD::VFPCLASS, and llvm::X86ISD::VFPCLASSS.

Referenced by LowerINTRINSIC_WO_CHAIN().

static SDValue getScalarValueForVectorElement ( SDValue  V,
int  Idx,
SelectionDAG DAG 
)
static
static SDValue getScatterNode ( unsigned  Opc,
SDValue  Op,
SelectionDAG DAG,
SDValue  Src,
SDValue  Mask,
SDValue  Base,
SDValue  Index,
SDValue  ScaleOp,
SDValue  Chain,
const X86Subtarget Subtarget 
)
static
static int getSEHRegistrationNodeSize ( const Function Fn)
static
static SDValue getSETCC ( X86::CondCode  Cond,
SDValue  EFLAGS,
const SDLoc dl,
SelectionDAG DAG 
)
static
static SDValue getShuffleScalarElt ( SDNode N,
unsigned  Index,
SelectionDAG DAG,
unsigned  Depth 
)
static
static SDValue getShuffleVectorZeroOrUndef ( SDValue  V2,
int  Idx,
bool  IsZero,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Return a vector_shuffle of the specified vector of zero or undef vector.

This produces a shuffle where the low element of V2 is swizzled into the zero/undef vector, landing at element Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).

Definition at line 5153 of file X86ISelLowering.cpp.

References llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), and i.

Referenced by llvm::X86TargetLowering::BuildFILD().

static bool getTargetConstantBitsFromNode ( SDValue  Op,
unsigned  EltSizeInBits,
SmallBitVector UndefElts,
SmallVectorImpl< APInt > &  EltBits 
)
static
static const Constant* getTargetConstantFromNode ( SDValue  Op)
static
static bool getTargetShuffleMask ( SDNode N,
MVT  VT,
bool  AllowSentinelZero,
SmallVectorImpl< SDValue > &  Ops,
SmallVectorImpl< int > &  Mask,
bool IsUnary 
)
static

Calculates the shuffle mask corresponding to the target-specific opcode.

If the mask could be calculated, returns it in Mask, returns the shuffle operands in Ops, and returns true. Sets IsUnary to true if only one source is used. Note that this will set IsUnary for shuffles which use a single input multiple times, and in those cases it will adjust the mask to only have indices within that single input. It is an error to call this with non-empty Mask/Ops vectors.

Definition at line 5393 of file X86ISelLowering.cpp.

References llvm::any_of(), assert(), llvm::X86ISD::BLENDI, llvm::DecodeBLENDMask(), llvm::DecodeINSERTPSMask(), llvm::DecodeMOVDDUPMask(), llvm::DecodeMOVHLPSMask(), llvm::DecodeMOVLHPSMask(), llvm::DecodeMOVSHDUPMask(), llvm::DecodeMOVSLDUPMask(), llvm::DecodePALIGNRMask(), llvm::DecodePSHUFBMask(), llvm::DecodePSHUFHWMask(), llvm::DecodePSHUFLWMask(), llvm::DecodePSHUFMask(), llvm::DecodePSLLDQMask(), llvm::DecodePSRLDQMask(), llvm::DecodeScalarMoveMask(), llvm::DecodeSHUFPMask(), llvm::DecodeUNPCKHMask(), llvm::DecodeUNPCKLMask(), llvm::DecodeVectorBroadcast(), llvm::DecodeVPERM2X128Mask(), llvm::DecodeVPERMIL2PMask(), llvm::DecodeVPERMILPMask(), llvm::DecodeVPERMMask(), llvm::DecodeVPERMV3Mask(), llvm::DecodeVPERMVMask(), llvm::DecodeVPPERMMask(), llvm::DecodeZeroMoveLowMask(), llvm::SmallVectorBase::empty(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), getTargetConstantFromNode(), getTargetShuffleMaskIndices(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::i8, llvm::X86ISD::INSERTPS, llvm_unreachable, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPD, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVLPD, llvm::X86ISD::MOVLPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::X86ISD::SHUFP, llvm::SM_SentinelZero, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMIV3, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VZEXT_MOVL.

Referenced by getPSHUFShuffleMask(), getShuffleScalarElt(), setTargetShuffleZeroElements(), and XFormVExtractWithShuffleIntoLoad().

static bool getTargetShuffleMaskIndices ( SDValue  MaskNode,
unsigned  MaskEltSizeInBits,
SmallVectorImpl< uint64_t > &  RawMask 
)
static
static SDValue getTargetVShiftByConstNode ( unsigned  Opc,
const SDLoc dl,
MVT  VT,
SDValue  SrcOp,
uint64_t  ShiftAmt,
SelectionDAG DAG 
)
static
static SDValue getTargetVShiftNode ( unsigned  Opc,
const SDLoc dl,
MVT  VT,
SDValue  SrcOp,
SDValue  ShAmt,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue GetTLSADDR ( SelectionDAG DAG,
SDValue  Chain,
GlobalAddressSDNode GA,
SDValue InFlag,
const EVT  PtrVT,
unsigned  ReturnReg,
unsigned char  OperandFlags,
bool  LocalDynamic = false 
)
static
static int getUnderlyingExtractedFromVec ( SDValue ExtractedFromVec,
SDValue  ExtIdx 
)
static

For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.

Modifies ExtractedFromVec to the real vector and returns the real index.

Definition at line 6604 of file X86ISelLowering.cpp.

References assert(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), and isUndefOrInRange().

Referenced by buildFromShuffleMostly().

static SDValue getUnpackh ( SelectionDAG DAG,
const SDLoc dl,
MVT  VT,
SDValue  V1,
SDValue  V2 
)
static

Returns a vector_shuffle node for an unpackh operation.

Definition at line 5142 of file X86ISelLowering.cpp.

References createUnpackShuffleMask(), llvm::SelectionDAG::getVectorShuffle(), and llvm::BitmaskEnumDetail::Mask().

Referenced by LowerAVXExtend().

static SDValue getUnpackl ( SelectionDAG DAG,
const SDLoc dl,
MVT  VT,
SDValue  V1,
SDValue  V2 
)
static

Returns a vector_shuffle node for an unpackl operation.

Definition at line 5134 of file X86ISelLowering.cpp.

References createUnpackShuffleMask(), llvm::SelectionDAG::getVectorShuffle(), and llvm::BitmaskEnumDetail::Mask().

Referenced by llvm::X86TargetLowering::BuildFILD(), and LowerAVXExtend().

static unsigned getV4X86ShuffleImm ( ArrayRef< int >  Mask)
static

Get a 4-lane 8-bit shuffle immediate for a mask.

This helper function produces an 8-bit shuffle immediate corresponding to the ubiquitous shuffle encoding scheme used in x86 instructions for shuffling 4 lanes. It can be used with most of the PSHUF instructions for example.

NB: We rely heavily on "undef" masks preserving the input lane.

Definition at line 7997 of file X86ISelLowering.cpp.

References assert(), and llvm::ArrayRef< T >::size().

Referenced by getV4X86ShuffleImm8ForMask(), matchBinaryPermuteVectorShuffle(), and matchUnaryPermuteVectorShuffle().

static SDValue getV4X86ShuffleImm8ForMask ( ArrayRef< int >  Mask,
SDLoc  DL,
SelectionDAG DAG 
)
static
static SDValue getv64i1Argument ( CCValAssign VA,
CCValAssign NextVA,
SDValue Root,
SelectionDAG DAG,
const SDLoc Dl,
const X86Subtarget Subtarget,
SDValue InFlag = nullptr 
)
static

Reads two 32 bit registers and creates a 64 bit mask value.

Parameters
VAThe current 32 bit value that need to be assigned.
NextVAThe next 32 bit value that need to be assigned.
RootThe parent DAG node.
[in,out]InFlagRepresents SDvalue in the parent DAG node for glue purposes. In the case the DAG is already using physical register instead of virtual, we should glue our new SDValue to InFlag SDvalue.
Returns
a new SDvalue of size 64bit.

Definition at line 2402 of file X86ISelLowering.cpp.

References llvm::MachineFunction::addLiveIn(), assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getCopyFromReg(), llvm::CCValAssign::getLocReg(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::CCValAssign::getValVT(), llvm::X86Subtarget::hasBWI(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::X86Subtarget::is32Bit(), llvm::CCValAssign::isRegLoc(), llvm::MipsISD::Lo, llvm::MVT::v32i1, and llvm::MVT::v64i1.

static SDValue getVectorMaskingNode ( SDValue  Op,
SDValue  Mask,
SDValue  PreservedSrc,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue getVShift ( bool  isLeft,
EVT  VT,
SDValue  SrcOp,
unsigned  NumBits,
SelectionDAG DAG,
const TargetLowering TLI,
const SDLoc dl 
)
static
static SDValue getZeroVector ( MVT  VT,
const X86Subtarget Subtarget,
SelectionDAG DAG,
const SDLoc dl 
)
static
static bool hasFPCMov ( unsigned  X86CC)
static

Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.

Definition at line 4323 of file X86ISelLowering.cpp.

References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, and llvm::X86::COND_P.

Referenced by combineCMov().

static bool hasNonFlagsUse ( SDValue  Op)
static
static SDValue insert128BitVector ( SDValue  Result,
SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
const SDLoc dl 
)
static
static SDValue insert1BitVector ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue insert256BitVector ( SDValue  Result,
SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
const SDLoc dl 
)
static
static SDValue insertSubVector ( SDValue  Result,
SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
const SDLoc dl,
unsigned  vectorWidth 
)
static
static bool is128BitLaneCrossingShuffleMask ( MVT  VT,
ArrayRef< int >  Mask 
)
static

Test whether there are elements crossing 128-bit lanes in this shuffle mask.

X86 divides up its shuffles into in-lane and cross-lane shuffle operations and we routinely test for these.

Definition at line 7829 of file X86ISelLowering.cpp.

References llvm::MVT::getScalarSizeInBits(), i, and llvm::ArrayRef< T >::size().

Referenced by combineX86ShuffleChain(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerVectorShuffleByMerging128BitLanes(), and matchUnaryPermuteVectorShuffle().

static bool is128BitLaneRepeatedShuffleMask ( MVT  VT,
ArrayRef< int >  Mask,
SmallVectorImpl< int > &  RepeatedMask 
)
static
static bool is256BitLaneRepeatedShuffleMask ( MVT  VT,
ArrayRef< int >  Mask,
SmallVectorImpl< int > &  RepeatedMask 
)
static

Test whether a shuffle mask is equivalent within each 256-bit lane.

Definition at line 7886 of file X86ISelLowering.cpp.

References isRepeatedShuffleMask().

Referenced by lowerV8F64VectorShuffle(), lowerV8I64VectorShuffle(), and matchUnaryPermuteVectorShuffle().

static bool isAddSub ( const BuildVectorSDNode BV,
const X86Subtarget Subtarget,
SelectionDAG DAG,
SDValue Opnd0,
SDValue Opnd1 
)
static
static bool isAddSub ( SDNode N,
const X86Subtarget Subtarget,
SDValue Opnd0,
SDValue Opnd1 
)
static

Returns true iff the shuffle node N can be replaced with ADDSUB operation.

If true is returned then the operands of ADDSUB operation are written to the parameters Opnd0 and Opnd1.

We combine shuffle to ADDSUB directly on the abstract vector shuffle nodes so it is easier to generically match. We also insert dummy vector shuffle nodes for the operands which explicitly discard the lanes which are unused by this operation to try to flow through the rest of the combiner the fact that they're unused.

Definition at line 28046 of file X86ISelLowering.cpp.

References llvm::ArrayRef< T >::begin(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::ArrayRef< T >::end(), llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE3(), isShuffleEquivalent(), llvm::BitmaskEnumDetail::Mask(), N, std::swap(), llvm::MVT::v16f32, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v8f32, llvm::MVT::v8f64, and llvm::ISD::VECTOR_SHUFFLE.

static bool isAndOrOfSetCCs ( SDValue  Op,
unsigned Opc 
)
static

Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR.

Definition at line 17945 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::ISD::OR, and llvm::X86ISD::SETCC.

Referenced by combineCompareEqual().

static bool isCMOVPseudo ( MachineInstr MI)
static

Definition at line 24783 of file X86ISelLowering.cpp.

References llvm::MachineInstr::getOpcode().

static bool isFMAddSub ( const X86Subtarget Subtarget,
SelectionDAG DAG,
SDValue Opnd0,
SDValue Opnd1,
SDValue Opnd2 
)
static

Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB(Opnd0, Opnd1) into FMADDSUB(x, y, Opnd1).

If (and only if) true is returned, the operands of FMADDSUB are written to parameters Opnd0, Opnd1, Opnd2.

Prior to calling this function it should be known that there is some SDNode that potentially can be replaced with an X86ISD::ADDSUB operation using Opnd0 and Opnd1 as operands. Also, this method is called before replacement of such SDNode with ADDSUB operation. Thus the number of Opnd0 uses is expected to be equal to 2. For example, this function may be called for the following IR: AB = fmul fast <2 x double> A, B Sub = fsub fast <2 x double> AB, C Add = fadd fast <2 x double> AB, C Addsub = shufflevector <2 x double> Sub, <2 x double> Add, <2 x i32> <i32 0, i32 3> There is a def for Addsub here, which potentially can be replaced by X86ISD::ADDSUB operation: Addsub = X86ISD::ADDSUB AB, C and such ADDSUB can further be replaced with FMADDSUB: Addsub = FMADDSUB A, B, C.

The main reason why this method is called before the replacement of the recognized ADDSUB idiom with ADDSUB operation is that such replacement is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit FMADDSUB is.

Definition at line 7095 of file X86ISelLowering.cpp.

References llvm::TargetOptions::AllowFPOpFusion, llvm::FPOpFusion::Fast, llvm::ISD::FMUL, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTarget(), llvm::X86Subtarget::hasAnyFMA(), llvm::TargetMachine::Options, llvm::TargetOptions::UnsafeFPMath, and llvm::SDNode::use_size().

Referenced by combineShuffleToAddSubOrFMAddSub(), and lowerToAddSubOrFMAddSub().

static SDValue isFNEG ( SDNode N)
static

Returns the negated value if the node N flips sign of FP value.

FP-negation node may have different forms: FNEG(x) or FXOR (x, 0x80000000). AVX512F does not have FXOR, so FNEG is lowered as (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))). In this case we go though all bitcasts.

Definition at line 32482 of file X86ISelLowering.cpp.

References C, llvm::ISD::FNEG, llvm::X86ISD::FXOR, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::APInt::getSignBit(), getTargetConstantFromNode(), llvm::SDValue::getValueType(), llvm::EVT::isFloatingPoint(), peekThroughBitcasts(), llvm::X86ISD::VBROADCAST, and llvm::ISD::XOR.

Referenced by combineFMA(), combineFneg(), combineFOr(), and combineXor().

static bool isFRClass ( const TargetRegisterClass RC)
static

Check if RC is a vector register class.

I.e., FR* / VR* or one of their variant.

Definition at line 34520 of file X86ISelLowering.cpp.

References llvm::TargetRegisterClass::hasSuperClassEq().

Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().

static bool isGRClass ( const TargetRegisterClass RC)
static

Check if RC is a general purpose register class.

I.e., GR* or one of their variant.

Definition at line 34510 of file X86ISelLowering.cpp.

References llvm::TargetRegisterClass::hasSuperClassEq().

Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().

static bool isHorizontalBinOp ( const BuildVectorSDNode N,
unsigned  Opcode,
SelectionDAG DAG,
unsigned  BaseIdx,
unsigned  LastIdx,
SDValue V0,
SDValue V1 
)
static

Return true if N implements a horizontal binop and return the operands for the horizontal binop into V0 and V1.

This is a helper function of LowerToHorizontalOp(). This function checks that the build_vector N in input implements a horizontal operation. Parameter Opcode defines the kind of horizontal operation to match. For example, if Opcode is equal to ISD::ADD, then this function checks if N implements a horizontal arithmetic add; if instead Opcode is equal to ISD::SUB, then this function checks if this is a horizontal arithmetic sub.

This function only analyzes elements of N whose indices are in range [BaseIdx, LastIdx).

Definition at line 6814 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), i, llvm::SDValue::isUndef(), llvm::SDNode::isUndef(), and llvm::EVT::isVector().

Referenced by combineAdd(), combineFaddFsub(), combineSub(), and LowerToHorizontalOp().

static bool isHorizontalBinOp ( SDValue LHS,
SDValue RHS,
bool  IsCommutative 
)
static

Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS.

A horizontal operation performs the binary operation on successive elements of its first operand, then on successive elements of its second operand, returning the resulting values in a vector. For example, if A = < float a0, float a1, float a2, float a3 > and B = < float b0, float b1, float b2, float b3 > then the result of doing a horizontal operation on A and B is A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >. In short, LHS and RHS are inspected to see if LHS op RHS is of the form A horizontal-op B, for some already available A and B, and if so then LHS is set to A, RHS to B, and the routine returns 'true'. Note that the binary operation should have the property that if one of the operands is UNDEF then the result is UNDEF.

Definition at line 32057 of file X86ISelLowering.cpp.

References A, assert(), B, llvm::SmallVectorTemplateCommon< T >::begin(), llvm::ArrayRef< T >::begin(), C, llvm::ShuffleVectorSDNode::commuteMask(), D, llvm::ArrayRef< T >::end(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), i, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::SDValue::isUndef(), and llvm::ISD::VECTOR_SHUFFLE.

static bool isNonZeroElementsInOrder ( const SmallBitVector  Zeroable,
ArrayRef< int >  Mask,
const EVT VectorType,
bool IsZeroSideLeft 
)
static
static bool isNoopShuffleMask ( ArrayRef< int >  Mask)
static

Tiny helper function to identify a no-op mask.

This is a somewhat boring predicate function. It checks whether the mask array input, which is assumed to be a single-input shuffle mask of the kind used by the X86 shuffle instructions (not a fully general ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an in-place shuffle are 'no-op's.

Definition at line 7815 of file X86ISelLowering.cpp.

References assert(), i, and llvm::ArrayRef< T >::size().

Referenced by combineTargetShuffle(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), and lowerVectorShuffleAsPermuteAndUnpack().

static bool isNullFPScalarOrVectorConst ( SDValue  V)
static
static bool isRepeatedShuffleMask ( unsigned  LaneSizeInBits,
MVT  VT,
ArrayRef< int >  Mask,
SmallVectorImpl< int > &  RepeatedMask 
)
static

Test whether a shuffle mask is equivalent within each sub-lane.

This checks a shuffle mask to see if it is performing the same lane-relative shuffle in each sub-lane. This trivially implies that it is also not lane-crossing. It may however involve a blend from the same lane of a second vector.

The specific repeated shuffle mask is populated in RepeatedMask, as it is non-trivial to compute in the face of undef lanes. The representation is suitable for use with existing 128-bit shuffles as entries from the second vector have been remapped to [LaneSize, 2*LaneSize).

Definition at line 7849 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::MVT::getScalarSizeInBits(), i, llvm::ArrayRef< T >::size(), and llvm::SM_SentinelUndef.

Referenced by is128BitLaneRepeatedShuffleMask(), and is256BitLaneRepeatedShuffleMask().

static bool isRepeatedTargetShuffleMask ( unsigned  LaneSizeInBits,
MVT  VT,
ArrayRef< int >  Mask,
SmallVectorImpl< int > &  RepeatedMask 
)
static

Test whether a target shuffle mask is equivalent within each sub-lane.

Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.

Definition at line 7893 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::MVT::getScalarSizeInBits(), i, isUndefOrZero(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by matchBinaryPermuteVectorShuffle().

static bool isSequentialOrUndefInRange ( ArrayRef< int >  Mask,
unsigned  Pos,
unsigned  Size,
int  Low 
)
static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size].

or is undef.

Definition at line 4516 of file X86ISelLowering.cpp.

References i, and isUndefOrEqual().

Referenced by combineX86ShuffleChain(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), lowerVectorShuffleWithUndefHalf(), matchUnaryPermuteVectorShuffle(), and matchVectorShuffleAsShift().

static bool isSequentialOrUndefOrZeroInRange ( ArrayRef< int >  Mask,
unsigned  Pos,
unsigned  Size,
int  Low 
)
static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero.

Definition at line 4527 of file X86ISelLowering.cpp.

References i, and isUndefOrZero().

Referenced by combineX86ShuffleChain(), and matchBinaryPermuteVectorShuffle().

static bool isShuffleEquivalent ( SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
ArrayRef< int >  ExpectedMask 
)
static

Checks whether a shuffle mask is equivalent to an explicit list of arguments.

This is a fast way to test a shuffle mask against a fixed pattern:

if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }

It returns true if the mask is exactly as wide as the argument list, and each element of the mask is either -1 (signifying undef) or the value given in the argument.

Definition at line 7937 of file X86ISelLowering.cpp.

References assert(), llvm::dyn_cast(), i, llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.

Referenced by combineTargetShuffle(), isAddSub(), lowerV16F32VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2X128VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4X128VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerVectorShuffleAsBroadcast(), and lowerVectorShuffleWithUNPCK().

static bool isShuffleFoldableLoad ( SDValue  V)
static

Helper to test for a load that can be folded with x86 shuffles.

This is particularly important because the set of instructions varies significantly based on whether the operand is a load or not.

Definition at line 9306 of file X86ISelLowering.cpp.

References llvm::SDValue::getNode(), llvm::ISD::isNON_EXTLoad(), and peekThroughBitcasts().

Referenced by combineTargetShuffle(), lowerV2F64VectorShuffle(), and lowerVectorShuffleAsBroadcast().

static bool isShuffleMaskInputInPlace ( int  Input,
ArrayRef< int >  Mask 
)
static

Test whether the specified input (0 or 1) is in-place blended by the given mask.

This returns true if the elements from a particular input are already in the slot required by the given mask and require no permutation.

Definition at line 11910 of file X86ISelLowering.cpp.

References assert(), i, and llvm::ArrayRef< T >::size().

Referenced by lowerV4F64VectorShuffle(), and lowerV4I64VectorShuffle().

static bool isSingleSHUFPSMask ( ArrayRef< int >  Mask)
static

Test whether this can be lowered with a single SHUFPS instruction.

This is used to disable more specialized lowerings when the shufps lowering will happen to be efficient.

Definition at line 10070 of file X86ISelLowering.cpp.

References assert(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I32VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4I32VectorShuffle(), and lowerV8I32VectorShuffle().

static bool isSortedByValueNo ( const SmallVectorImpl< CCValAssign > &  ArgLocs)
static
static bool isTargetShuffle ( unsigned  Opcode)
static
static bool isTargetShuffleEquivalent ( ArrayRef< int >  Mask,
ArrayRef< int >  ExpectedMask 
)
static

Checks whether a target shuffle mask is equivalent to an explicit pattern.

The masks must be exactly the same width.

If an element in Mask matches SM_SentinelUndef (-1) then the corresponding value in ExpectedMask is always accepted. Otherwise the indices must match.

SM_SentinelZero is accepted as a valid negative index but must match in both.

Definition at line 7972 of file X86ISelLowering.cpp.

References i, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by matchBinaryVectorShuffle(), and matchUnaryVectorShuffle().

static bool isTargetShuffleVariableMask ( unsigned  Opcode)
static
static bool isTruncWithZeroHighBitsInput ( SDValue  V,
SelectionDAG DAG 
)
static
static bool isUndefInRange ( ArrayRef< int >  Mask,
unsigned  Pos,
unsigned  Size 
)
static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value.

Definition at line 4475 of file X86ISelLowering.cpp.

References i, and llvm::SM_SentinelUndef.

Referenced by lowerVectorShuffleAsSpecificZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), lowerVectorShuffleWithUndefHalf(), and matchBinaryPermuteVectorShuffle().

static bool isUndefOrEqual ( int  Val,
int  CmpVal 
)
static

Val is the undef sentinel value or equal to the specified value.

Definition at line 4464 of file X86ISelLowering.cpp.

References llvm::SM_SentinelUndef.

Referenced by isSequentialOrUndefInRange(), and matchUnaryVectorShuffle().

static bool isUndefOrInRange ( int  Val,
int  Low,
int  Hi 
)
static

Return true if Val is undef or if its value falls within the specified range (L, H].

Definition at line 4484 of file X86ISelLowering.cpp.

References llvm::MipsISD::Hi, and llvm::SM_SentinelUndef.

Referenced by getUnderlyingExtractedFromVec(), isUndefOrInRange(), matchBinaryPermuteVectorShuffle(), and matchUnaryPermuteVectorShuffle().

static bool isUndefOrInRange ( ArrayRef< int >  Mask,
int  Low,
int  Hi 
)
static

Return true if every element in Mask is undef or if its value falls within the specified range (L, H].

Definition at line 4490 of file X86ISelLowering.cpp.

References isUndefOrInRange().

static bool isUndefOrZero ( int  Val)
static
static bool isUndefOrZeroInRange ( ArrayRef< int >  Mask,
unsigned  Pos,
unsigned  Size 
)
static

Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero.

Definition at line 4537 of file X86ISelLowering.cpp.

References i, and isUndefOrZero().

Referenced by matchBinaryPermuteVectorShuffle(), and matchUnaryVectorShuffle().

static bool isUndefOrZeroOrInRange ( int  Val,
int  Low,
int  Hi 
)
static

Return true if Val is undef, zero or if its value falls within the specified range (L, H].

Definition at line 4500 of file X86ISelLowering.cpp.

References llvm::MipsISD::Hi, and isUndefOrZero().

Referenced by combineX86ShuffleChain(), and isUndefOrZeroOrInRange().

static bool isUndefOrZeroOrInRange ( ArrayRef< int >  Mask,
int  Low,
int  Hi 
)
static

Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H].

Definition at line 4506 of file X86ISelLowering.cpp.

References isUndefOrZeroOrInRange().

static bool isUseOfShuffle ( SDNode N)
static

Definition at line 6402 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, isTargetShuffle(), and llvm::SDNode::uses().

Referenced by LowerVectorBroadcast().

static bool isVEXTRACTIndex ( SDNode N,
unsigned  vecWidth 
)
static

Return true if the specified EXTRACT_SUBVECTOR operand specifies a vector extract that is suitable for instruction that extract 128 or 256 bit vectors.

Definition at line 4631 of file X86ISelLowering.cpp.

References assert(), llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), and llvm::SDNode::getSimpleValueType().

Referenced by llvm::X86::isVEXTRACT128Index(), and llvm::X86::isVEXTRACT256Index().

static bool isVINSERTIndex ( SDNode N,
unsigned  vecWidth 
)
static

Return true if the specified INSERT_SUBVECTOR operand specifies a subvector insert that is suitable for input to insertion of 128 or 256-bit subvectors.

Definition at line 4650 of file X86ISelLowering.cpp.

References assert(), llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), and llvm::SDNode::getSimpleValueType().

Referenced by llvm::X86::isVINSERT128Index(), and llvm::X86::isVINSERT256Index().

static bool isX86CCUnsigned ( unsigned  X86CC)
static
static bool isX86LogicalCmp ( SDValue  Op)
static
static bool isXor1OfSetCC ( SDValue  Op)
static

Return true if node is an ISD::XOR of a X86ISD::SETCC and 1 and that the SETCC node has a single use.

Definition at line 17957 of file X86ISelLowering.cpp.

References llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isOneConstant(), llvm::X86ISD::SETCC, and llvm::ISD::XOR.

static SDValue lower128BitVectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
MVT  VT,
SDValue  V1,
SDValue  V2,
const SmallBitVector Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Dispatching routine to lower various 128-bit x86 vector shuffles.

This routine breaks down the specific type of 128-bit shuffle and dispatches to the lowering routines accordingly.

Definition at line 11366 of file X86ISelLowering.cpp.

References llvm_unreachable, lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4I32VectorShuffle(), lowerV8I16VectorShuffle(), llvm::MVT::SimpleTy, llvm::MVT::v16i8, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4i32, and llvm::MVT::v8i16.

Referenced by lowerVectorShuffle().

static SDValue lower1BitVectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
MVT  VT,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lower256BitVectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
MVT  VT,
SDValue  V1,
SDValue  V2,
const SmallBitVector Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue Lower256IntArith ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue Lower256IntVSETCC ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue lower512BitVectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
MVT  VT,
SDValue  V1,
SDValue  V2,
const SmallBitVector Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

High-level routine to lower various 512-bit x86 vector shuffles.

This routine either breaks down the specific type of a 512-bit x86 vector shuffle or splits it into two 256-bit shuffles and fuses the results back together based on the available instructions.

Definition at line 13171 of file X86ISelLowering.cpp.

References assert(), llvm::count_if(), llvm::X86Subtarget::hasAVX512(), llvm_unreachable, lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV32I16VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I64VectorShuffle(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsElementInsertion(), llvm::MVT::SimpleTy, llvm::ArrayRef< T >::size(), llvm::MVT::v16f32, llvm::MVT::v16i32, llvm::MVT::v32i16, llvm::MVT::v64i8, llvm::MVT::v8f64, and llvm::MVT::v8i64.

Referenced by lowerVectorShuffle().

static SDValue Lower512IntArith ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerADD ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerADDC_ADDE_SUBC_SUBE ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerADJUST_TRAMPOLINE ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerAndToBT ( SDValue  And,
ISD::CondCode  CC,
const SDLoc dl,
SelectionDAG DAG 
)
static
static SDValue LowerANY_EXTEND ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerAsSplatVectorLoad ( SDValue  SrcOp,
MVT  VT,
const SDLoc dl,
SelectionDAG DAG 
)
static
static SDValue LowerATOMIC_FENCE ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerATOMIC_STORE ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue lowerAtomicArith ( SDValue  N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue lowerAtomicArithWithLOCK ( SDValue  N,
SelectionDAG DAG 
)
static
static SDValue LowerAVXCONCAT_VECTORS ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerAVXExtend ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue LowerBITCAST ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerBITREVERSE ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerBITREVERSE_XOP ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerBoolVSETCC_AVX512 ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue lowerBuildVectorToBitOp ( BuildVectorSDNode Op,
SelectionDAG DAG 
)
static

If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors.

NOTE: Its not in our interest to start make a general purpose vectorizer from this, but enough scalar bit operations are created from the later legalization + scalarization stages to need basic support.

Definition at line 7281 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), i, llvm::TargetLoweringBase::isOperationLegalOrPromote(), llvm::SDNode::ops(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), and llvm::ISD::XOR.

static SDValue LowerBuildVectorv16i8 ( SDValue  Op,
unsigned  NonZeros,
unsigned  NumNonZero,
unsigned  NumZero,
SelectionDAG DAG,
const X86Subtarget Subtarget,
const TargetLowering TLI 
)
static
static SDValue LowerBuildVectorv4x32 ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget,
const TargetLowering TLI 
)
static
static SDValue LowerBuildVectorv8i16 ( SDValue  Op,
unsigned  NonZeros,
unsigned  NumNonZero,
unsigned  NumZero,
SelectionDAG DAG,
const X86Subtarget Subtarget,
const TargetLowering TLI 
)
static
static SDValue LowerCMP_SWAP ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerCONCAT_VECTORS ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerCONCAT_VECTORSvXi1 ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerCTLZ ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerCTPOP ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerCTTZ ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerEXTEND_VECTOR_INREG ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerExtended1BitVectorLoad ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerExtendedLoad ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 17762 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::ISD::EXTLOAD, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getExtLoad(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SelectionDAG::getSignExtendVectorInReg(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), i, llvm::MVT::i1, llvm::MVT::i8, llvm::ISD::INSERT_VECTOR_ELT, llvm::MVT::integer_valuetypes(), llvm::MVT::isInteger(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::MVT::isVector(), llvm::SPII::Load, LowerExtended1BitVectorLoad(), llvm::MVT::Other, Ptr, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::TokenFactor, and llvm::X86ISD::VSEXT.

Referenced by llvm::X86TargetLowering::LowerOperation().

static SDValue LowerEXTRACT_SUBVECTOR ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerEXTRACT_VECTOR_ELT_SSE4 ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerFABSorFNEG ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerFCOPYSIGN ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerFGETSIGN ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerFP_EXTEND ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerFSINCOS ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerHorizontalByteSum ( SDValue  V,
MVT  VT,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Compute the horizontal sum of bytes in V for the elements of VT.

Requires V to be a byte vector and VT to be an integer vector type with wider elements than V's type. The width of the elements of VT determines how many bytes of V are summed horizontally to produce each element of the result.

Definition at line 22395 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getZeroVector(), High, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86ISD::PACKUS, llvm::X86ISD::PSADBW, llvm::ISD::SHL, llvm::ISD::SRL, llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by LowerVectorCTPOPBitmath(), and LowerVectorCTPOPInRegLUT().

static SDValue LowerINSERT_SUBVECTOR ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerINTRINSIC_W_CHAIN ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 19883 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::ADX, assert(), llvm::X86ISD::CMOV, llvm::COMPRESS_TO_MEM, llvm::X86::COND_B, llvm::X86::COND_NE, llvm::MachineFrameInfo::CreateFixedObject(), llvm::Depth, llvm::dyn_cast(), EmitMaskedTruncSStore(), EmitTruncSStore(), llvm::EXPAND_FROM_MEM, llvm::GATHER, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), getExtendedControlRegister(), llvm::X86MachineFunctionInfo::getFAIndex(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), getGatherNode(), llvm::MachineFunction::getInfo(), llvm::getIntrinsicWithChain(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMaskedStore(), getMaskNode(), llvm::TargetMachine::getMCAsmInfo(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), getPrefetchNode(), getReadPerformanceCounter(), getReadTimeStampCounter(), llvm::X86Subtarget::getRegisterInfo(), llvm::X86TargetLowering::getReturnAddressFrameIndex(), getScatterNode(), getSETCC(), llvm::SDValue::getSimpleValueType(), llvm::X86RegisterInfo::getSlotSize(), llvm::SelectionDAG::getStore(), llvm::MachineFunction::getTarget(), llvm::SelectionDAG::getTruncStore(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::Glue, llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::isAllOnesConstant(), isValid(), llvm::X86::isZeroNode(), llvm_unreachable, MarkEHGuard(), MarkEHRegistrationNode(), llvm::ISD::MERGE_VALUES, llvm::ISD::NON_EXTLOAD, llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, llvm::MVT::Other, llvm::PREFETCH, llvm::RDPMC, llvm::RDRAND, llvm::RDSEED, llvm::RDTSC, Results, llvm::MipsISD::Ret, llvm::SCATTER, llvm::X86MachineFunctionInfo::setFAIndex(), llvm::MachineFrameInfo::setFrameAddressIsTaken(), llvm::MachineFrameInfo::setHasCopyImplyingStackAdjustment(), llvm::MachineFrameInfo::setReturnAddressIsTaken(), llvm::TRUNCATE_TO_MEM_VI16, llvm::TRUNCATE_TO_MEM_VI32, llvm::TRUNCATE_TO_MEM_VI8, llvm::IntrinsicData::Type, llvm::MCAsmInfo::usesWindowsCFI(), llvm::TargetLowering::verifyReturnAddressArgumentIsConstant(), llvm::X86ISD::VTRUNC, llvm::X86ISD::VTRUNCS, llvm::X86ISD::VTRUNCUS, llvm::XGETBV, llvm::XTEST, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

static SDValue LowerINTRINSIC_WO_CHAIN ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 18785 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::BRCST32x2_TO_VEC, llvm::BRCST_SUBVEC_TO_VEC, llvm::BROADCASTM, llvm::CMP_MASK, llvm::CMP_MASK_CC, llvm::CMP_MASK_SCALAR_CC, llvm::COMI, llvm::COMI_RM, llvm::COMPRESS_EXPAND_IN_REG, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_O, llvm::X86::COND_P, llvm::X86::COND_S, llvm::CONVERT_MASK_TO_VEC, llvm::CONVERT_TO_MASK, llvm::X86::CUR_DIRECTION, llvm::CVTPD2PS, llvm::CVTPD2PS_MASK, llvm::dyn_cast(), llvm::MVT::f32, llvm::MVT::f64, llvm::FIXUPIMM, llvm::FIXUPIMM_MASKZ, llvm::FIXUPIMMS, llvm::FIXUPIMMS_MASKZ, llvm::FMA_OP_MASK, llvm::FMA_OP_MASK3, llvm::FMA_OP_MASKZ, llvm::FMA_OP_SCALAR_MASK, llvm::FMA_OP_SCALAR_MASK3, llvm::FMA_OP_SCALAR_MASKZ, llvm::ISD::FP_ROUND, llvm::FPCLASS, llvm::FPCLASSS, llvm::X86ISD::FSETCCM, llvm::X86ISD::FSETCCM_RND, llvm::X86RegisterInfo::getBaseRegister(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MachineModuleInfo::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MachineFunction::getFunctionNumber(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::getIntrinsicWithoutChain(), llvm::SelectionDAG::getMachineFunction(), getMaskNode(), llvm::SelectionDAG::getMCSymbol(), llvm::MachineFunction::getMMI(), llvm::Value::getName(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::MCContext::getOrCreateLSDASymbol(), llvm::TargetLoweringBase::getPointerTy(), llvm::X86RegisterInfo::getPtrSizedFrameRegister(), llvm::GlobalValue::getRealLinkageName(), llvm::X86Subtarget::getRegisterInfo(), getScalarMaskingNode(), llvm::MVT::getScalarType(), getSETCC(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), getVectorMaskingNode(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::X86RegisterInfo::hasBasePointer(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::INTR_TYPE_1OP, llvm::INTR_TYPE_1OP_MASK, llvm::INTR_TYPE_1OP_MASK_RM, llvm::INTR_TYPE_2OP, llvm::INTR_TYPE_2OP_IMM8_MASK, llvm::INTR_TYPE_2OP_MASK, llvm::INTR_TYPE_2OP_MASK_RM, llvm::INTR_TYPE_3OP, llvm::INTR_TYPE_3OP_IMM8_MASK, llvm::INTR_TYPE_3OP_MASK, llvm::INTR_TYPE_3OP_MASK_RM, llvm::INTR_TYPE_3OP_SCALAR_MASK_RM, llvm::INTR_TYPE_4OP, llvm::INTR_TYPE_SCALAR_MASK, llvm::INTR_TYPE_SCALAR_MASK_RM, llvm::MVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::isAllOnesConstant(), llvm::X86ISD::KORTEST, llvm::KUNPCK, LLVM_FALLTHROUGH, llvm_unreachable, llvm::BitmaskEnumDetail::Mask(), llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, llvm::ISD::OR, llvm::X86ISD::PCMPESTRI, llvm::X86ISD::PCMPISTRI, llvm::X86ISD::PTEST, recoverFramePointer(), llvm::report_fatal_error(), llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::TERLOG_OP_MASK, llvm::TERLOG_OP_MASKZ, llvm::X86ISD::TESTP, llvm::ISD::TRUNCATE, llvm::IntrinsicData::Type, llvm::MVT::v16i1, llvm::VPERM_2OP_MASK, llvm::VPERM_3OP_MASK, llvm::VPERM_3OP_MASKZ, llvm::X86ISD::VPERMV, llvm::VSHIFT, llvm::X86ISD::Wrapper, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation(), and llvm::X86TargetLowering::ReplaceNodeResults().

static SDValue LowerIntVSETCC_AVX512 ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue lowerMasksToReg ( const SDValue ValArg,
const EVT ValLoc,
const SDLoc Dl,
SelectionDAG DAG 
)
static
static SDValue LowerMGATHER ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerMINMAX ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerMLOAD ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerMSCATTER ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerMSTORE ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerMUL ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerMUL_LOHI ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerMULH ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerREADCYCLECOUNTER ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerRegToMasks ( const SDValue ValArg,
const EVT ValVT,
const EVT ValLoc,
const SDLoc Dl,
SelectionDAG DAG 
)
static

The function will lower a register of various sizes (8/16/32/64) to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1)

Returns
a DAG node contains the operand after lowering to mask type.

Definition at line 2454 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::EVT::getSimpleVT(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm_unreachable, llvm::MVT::SimpleTy, llvm::ISD::TRUNCATE, llvm::MVT::v16i1, llvm::MVT::v32i1, llvm::MVT::v64i1, and llvm::MVT::v8i1.

static SDValue LowerRotate ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerSCALAR_TO_VECTOR ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerScalarImmediateShift ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue LowerScalarVariableShift ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue LowerShift ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerShiftParts ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue lowerShuffleAsRepeatedMaskAndLanePermute ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes.

Definition at line 11924 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), i, is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), and llvm::SM_SentinelUndef.

Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().

static SDValue LowerSIGN_EXTEND ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerSIGN_EXTEND_AVX512 ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerSUB ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue lowerToAddSubOrFMAddSub ( const BuildVectorSDNode BV,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB node.

Definition at line 7120 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADDSUB, llvm::X86ISD::FMADDSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getSimpleValueType(), llvm::MVT::is512BitVector(), isAddSub(), and isFMAddSub().

static SDValue LowerToHorizontalOp ( const BuildVectorSDNode BV,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerToTLSExecModel ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT,
TLSModel::Model  model,
bool  is64Bit,
bool  isPIC 
)
static
static SDValue LowerToTLSGeneralDynamicModel32 ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT 
)
static
static SDValue LowerToTLSGeneralDynamicModel64 ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT 
)
static
static SDValue LowerToTLSLocalDynamicModel ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT,
bool  is64Bit 
)
static
static SDValue LowerTruncateToBT ( SDValue  Op,
ISD::CondCode  CC,
const SDLoc dl,
SelectionDAG DAG 
)
static
static SDValue LowerTruncateVecI1 ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue LowerTruncatingStore ( SDValue  StOp,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerUINT_TO_FP_v2i32 ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget,
SDLoc DL 
)
static
static SDValue lowerUINT_TO_FP_vXi32 ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue lowerV16F32VectorShuffle ( SDLoc  DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV16I16VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV16I32VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV16I8VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Generic lowering of v16i8 shuffles.

This is a hybrid strategy to lower v16i8 vectors. It first attempts to detect any complexity reducing interleaving. If that doesn't help, it uses UNPCK to spread the i8 elements across two i16-element vectors, and uses the existing lowering for v8i16 blends on each half, finally PACK-ing them back together.

Definition at line 11077 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), canLowerByDroppingEvenElements(), llvm::count_if(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), I, i, llvm::SDValue::isUndef(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBlendOfPSHUFBs(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPermuteAndUnpack(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), lowerVectorShuffleWithUNPCK(), llvm::BitmaskEnumDetail::Mask(), llvm::none_of(), llvm::X86ISD::PACKUS, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, and Zero.

Referenced by lower128BitVectorShuffle().

static SDValue lowerV2F64VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Handle lowering of 2-lane 64-bit floating point shuffles.

This is the basis function for the 2-lane 64-bit shuffles as we have full support for floating point shuffles but not integer shuffles. These instructions will incur a domain crossing penalty on some chips though so it is better to avoid lowering through this for integer vectors where possible.

Definition at line 9879 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getScalarValueForVectorElement(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i8, isShuffleEquivalent(), isShuffleFoldableLoad(), llvm::SDValue::isUndef(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVLPD, llvm::X86ISD::MOVSD, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::SHUFP, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::MVT::v2f64, and llvm::X86ISD::VPERMILPI.

Referenced by lower128BitVectorShuffle().

static SDValue lowerV2I64VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV2X128VectorShuffle ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV32I16VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV32I8VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV4F32VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV4F64VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV4I32VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV4I64VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV4X128VectorShuffle ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG 
)
static
static SDValue lowerV64I8VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV8F32VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV8F64VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV8I16GeneralSingleInputVectorShuffle ( const SDLoc DL,
MVT  VT,
SDValue  V,
MutableArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.

The lowering strategy is to try to form pairs of input lanes which are targeted at the same half of the final vector, and then use a dword shuffle to place them onto the right half, and finally unpack the paired lanes into their final position.

The exact breakdown of how to form these dword pairs and align them on the correct sides is really tricky. See the comments within the function for more of the details.

This code also handles repeated 128-bit lanes of v8i16 shuffles, but each lane must shuffle the exact same way. In fact, you must pass a v8 Mask to this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16 vector, form the analogous 128-bit 8-element Mask.

Definition at line 10381 of file X86ISelLowering.cpp.

References assert(), llvm::sys::path::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), llvm::MutableArrayRef< T >::begin(), llvm::count(), llvm::count_if(), llvm::SmallVectorTemplateCommon< T, typename >::data(), llvm::ArrayRef< T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::MutableArrayRef< T >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::find(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), i, llvm::MVT::i16, llvm::MVT::i32, llvm::is_contained(), isNoopShuffleMask(), llvm_unreachable, llvm::BitmaskEnumDetail::Mask(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::MutableArrayRef< T >::slice(), std::swap(), and llvm::MVT::v8i16.

Referenced by lowerV16I16VectorShuffle(), lowerV32I16VectorShuffle(), and lowerV8I16VectorShuffle().

static SDValue lowerV8I16VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Generic lowering of 8-lane i16 shuffles.

This handles both single-input shuffles and combined shuffle/blends with two inputs. The single input shuffles are immediately delegated to a dedicated lowering routine.

The blends are lowered in one of three fundamental ways. If there are few enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle of the input is significantly cheaper when lowered as an interleaving of the two inputs, try to interleave them. Otherwise, blend the low and high halves of the inputs separately (making them have relatively few inputs) and then concatenate them.

Definition at line 10893 of file X86ISelLowering.cpp.

References llvm::any_of(), assert(), llvm::ArrayRef< T >::begin(), llvm::count_if(), llvm::ArrayRef< T >::end(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBlendOfPSHUFBs(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPermuteAndUnpack(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), lowerVectorShuffleWithUNPCK(), llvm::ArrayRef< T >::size(), and llvm::MVT::v8i16.

Referenced by lower128BitVectorShuffle().

static SDValue lowerV8I32VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV8I64VectorShuffle ( const SDLoc DL,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVACOPY ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVectorAllZeroTest ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVectorBroadcast ( BuildVectorSDNode BVOp,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Attempt to use the vbroadcast instruction to generate a splat value for the following cases:

  1. A splat BUILD_VECTOR which uses: a. A single scalar load, or a constant. b. Repeated pattern of constants (e.g. <0,1,0,1> or <0,1,2,3,0,1,2,3>).
  2. A splat shuffle which uses a scalar_to_vector node which comes from a scalar load, or a constant.

The VBROADCAST node is returned when a pattern is found, or SDValue() otherwise.

Definition at line 6422 of file X86ISelLowering.cpp.

References assert(), llvm::APInt::bitsToDouble(), llvm::APInt::bitsToFloat(), llvm::ISD::Constant, llvm::ISD::ConstantFP, llvm::BitVector::count(), llvm::HexagonISD::CP, llvm::ConstantFP::get(), getAlignment(), llvm::SelectionDAG::getBitcast(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), getConstantVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::Type::getDoubleTy(), llvm::SelectionDAG::getEntryNode(), llvm::MVT::getFloatingPointVT(), llvm::Type::getFloatTy(), llvm::MachineFunction::getFunction(), llvm::Constant::getIntegerValue(), llvm::MVT::getIntegerVT(), llvm::Type::getIntNTy(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::TargetLoweringBase::getPointerTy(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::BuildVectorSDNode::getSplatValue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasVLX(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::X86Subtarget::is32Bit(), llvm::MVT::is512BitVector(), llvm::BuildVectorSDNode::isConstantSplat(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::SDNode::isOnlyUserOf(), isUseOfShuffle(), llvm::EVT::isVector(), llvm::Function::optForSize(), llvm::X86ISD::SUBV_BROADCAST, llvm::RegState::Undef, and llvm::X86ISD::VBROADCAST.

static SDValue LowerVectorCTLZ ( SDValue  Op,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVectorCTLZ_AVX512 ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerVectorCTLZInRegLUT ( SDValue  Op,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVectorCTPOP ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVectorCTPOPBitmath ( SDValue  Op,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVectorCTPOPInRegLUT ( SDValue  Op,
const SDLoc DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffle ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Top-level lowering for x86 vector shuffles.

This handles decomposition, canonicalization, and lowering of all x86 vector shuffles. Most of the specific lowering strategies are encapsulated above in helper routines. The canonicalization attempts to widen shuffles to involve fewer lanes of wider elements, consolidate symmetric patterns s.t. only one of the two inputs needs to be tested, etc.

Definition at line 13348 of file X86ISelLowering.cpp.

References llvm::SmallBitVector::all(), llvm::all_of(), assert(), canonicalizeShuffleMaskWithCommute(), canWidenShuffleElements(), computeZeroableShuffleElements(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getCommutedVectorShuffle(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::ShuffleVectorSDNode::getMask(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::i1, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm_unreachable, lower128BitVectorShuffle(), lower1BitVectorShuffle(), lower256BitVectorShuffle(), lower512BitVectorShuffle(), and llvm::NVPTX::PTXLdStInstCode::V2.

Referenced by llvm::X86TargetLowering::LowerOperation().

static SDValue lowerVectorShuffleAsBitBlend ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

Try to emit a blend instruction for a shuffle using bit math.

This is used as a fallback approach when first class blend instructions are unavailable. Currently it is only suitable for integer vectors, but could be generalized for floating point vectors if desirable.

Definition at line 8284 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), i, llvm::MVT::i64, llvm::MVT::isInteger(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ArrayRef< T >::size(), and Zero.

Referenced by lower256BitVectorShuffle(), lowerV16I8VectorShuffle(), and lowerV8I16VectorShuffle().

static SDValue lowerVectorShuffleAsBitMask ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsBlend ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Original,
const SmallBitVector Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to emit a blend instruction for a shuffle.

This doesn't do any checks for the availability of instructions for blending these values. It relies on the availability of the X86ISD::BLENDI pattern to be matched in the backend with the type given. What it does check for is that the shuffle mask is a blend, or convertible into a blend with zero.

Definition at line 8316 of file X86ISelLowering.cpp.

References assert(), llvm::ArrayRef< T >::begin(), llvm::X86ISD::BLENDI, llvm::ArrayRef< T >::end(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), i, llvm::MVT::i8, is128BitLaneRepeatedShuffleMask(), llvm::MVT::is128BitVector(), llvm::ISD::isBuildVectorAllZeros(), LLVM_FALLTHROUGH, llvm_unreachable, lowerVectorShuffleAsBitMask(), llvm::BitmaskEnumDetail::Mask(), llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::MVT::SimpleTy, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8f32, llvm::MVT::v8i16, llvm::MVT::v8i32, and llvm::ISD::VSELECT.

Referenced by lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV2X128VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().

static SDValue lowerVectorShuffleAsBlendAndPermute ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

Try to lower as a blend of elements from two inputs followed by a single-input permutation.

This matches the pattern where we can blend elements from two inputs and then reduce the shuffle to a single-input permutation.

Definition at line 8479 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), i, and llvm::ArrayRef< T >::size().

Referenced by lowerV4F32VectorShuffle(), and lowerVectorShuffleAsDecomposedShuffleBlend().

static SDValue lowerVectorShuffleAsBlendOfPSHUFBs ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SelectionDAG DAG,
bool V1InUse,
bool V2InUse 
)
static
static SDValue lowerVectorShuffleAsBroadcast ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to lower broadcast of a single element.

For convenience, this code also bundles all of the subtarget feature set filtering. While a little annoying to re-dispatch on type here, there isn't a convenient way to factor it out. FIXME: This is very similar to LowerVectorBroadcast - can we merge them?

Definition at line 9473 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getStoreSize(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ConstantSDNode::getZExtValue(), llvm::SDNode::hasAnyUseOfValue(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE3(), i, llvm::MVT::i64, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), llvm::MVT::isInteger(), isShuffleEquivalent(), isShuffleFoldableLoad(), llvm::MVT::isVector(), isVolatile(), lowerVectorShuffleAsTruncBroadcast(), MayFoldLoad(), llvm::X86ISD::MOVDDUP, Offset, llvm::MVT::Other, peekThroughBitcasts(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ArrayRef< T >::size(), llvm::ISD::TokenFactor, llvm::SelectionDAG::UpdateNodeOperands(), llvm::MVT::v2f64, llvm::MVT::v4f64, llvm::MVT::v4i64, and llvm::X86ISD::VBROADCAST.

Referenced by lower512BitVectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().

static SDValue lowerVectorShuffleAsByteRotate ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsDecomposedShuffleBlend ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

Generic routine to decompose a shuffle and blend into indepndent blends and permutes.

This matches the extremely common pattern for handling combined shuffle+blend operations on newer X86 ISAs where we have very fast blend operations. It will try to pick the best arrangement of shuffles and blends.

Definition at line 8513 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), i, isNoopShuffleMask(), lowerVectorShuffleAsBlendAndPermute(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerVectorShuffleAsSplitOrBlend().

static SDValue lowerVectorShuffleAsElementInsertion ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsInsertPS ( const SDLoc DL,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsLanePermuteAndBlend ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

Lower a vector shuffle crossing multiple 128-bit lanes as a permutation and blend of those lanes.

This essentially blends the out-of-lane inputs to each lane into the lane from a permuted copy of the vector. This lowering strategy results in four instructions in the worst case for a single-input cross lane shuffle which is lower than any other fully general cross-lane shuffle strategy I'm aware of. Special cases for each particular shuffle pattern should be handled prior to trying this lowering.

Definition at line 11580 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), i, llvm::MVT::i8, llvm::MVT::is256BitVector(), llvm::SDValue::isUndef(), llvm::ArrayRef< T >::size(), splitAndLowerVectorShuffle(), and llvm::X86ISD::VPERM2X128.

Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), and lowerV8F32VectorShuffle().

static SDValue lowerVectorShuffleAsPermuteAndUnpack ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.

This specifically targets cases where we end up with alternating between the two inputs, and so can permute them into something that feeds a single UNPCK instruction. Note that this routine only targets integer vectors because for floating point vectors we have a generalized SHUFPS lowering strategy that handles everything that doesn't exactly match an unpack, making this clever lowering unnecessary.

Definition at line 9769 of file X86ISelLowering.cpp.

References assert(), llvm::count_if(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), i, llvm::MVT::is128BitVector(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), llvm::SDValue::isUndef(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by lowerV16I8VectorShuffle(), lowerV4I32VectorShuffle(), and lowerV8I16VectorShuffle().

static SDValue lowerVectorShuffleAsRotate ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to lower a vector shuffle as a dword/qword rotation.

AVX512 has a VALIGND/VALIGNQ instructions that will do an arbitrary rotation of the concatenation of two vectors; This routine will try to generically lower a vector shuffle through such an pattern.

Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.

Definition at line 8713 of file X86ISelLowering.cpp.

References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::X86Subtarget::hasVLX(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MipsISD::Lo, matchVectorShuffleAsRotate(), llvm::NVPTX::PTXLdStInstCode::V2, and llvm::X86ISD::VALIGN.

Referenced by lowerV16I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().

static SDValue lowerVectorShuffleAsShift ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend ( const SDLoc DL,
MVT  VT,
int  Scale,
int  Offset,
bool  AnyExt,
SDValue  InputV,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Lower a vector shuffle as a zero or any extension.

Given a specific number of elements, element bit width, and extension stride, produce either a zero or any extension based on the available features of the subtarget. The extended elements are consecutive and begin and can start from an offseted element index in the input; to avoid excess shuffling the offset must either being in the bottom lane or at the start of a higher lane. All extended elements must be from the same lane.

Definition at line 8994 of file X86ISelLowering.cpp.

References assert(), llvm::MipsISD::Ext, extractSubVector(), llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), llvm::MipsISD::Hi, i, llvm::MVT::i8, llvm::MVT::is128BitVector(), isUndefInRange(), llvm::MipsISD::Lo, Offset, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, and llvm::X86ISD::VZEXT.

Referenced by lowerVectorShuffleAsZeroOrAnyExtend().

static SDValue lowerVectorShuffleAsSplitOrBlend ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

Either split a vector in halves or decompose the shuffles and the blend.

This is provided as a good fallback for many lowerings of non-single-input shuffles with more than one 128-bit lane. In those cases, we want to select between splitting the shuffle into 128-bit components and stitching those back together vs. extracting the single-input shuffles and blending those results.

Definition at line 11521 of file X86ISelLowering.cpp.

References assert(), llvm::SmallBitVector::count(), llvm::count(), llvm::MVT::getSizeInBits(), i, llvm::SDValue::isUndef(), lowerVectorShuffleAsDecomposedShuffleBlend(), llvm::SmallBitVector::resize(), llvm::ArrayRef< T >::size(), and splitAndLowerVectorShuffle().

Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), and lowerV8F32VectorShuffle().

static SDValue lowerVectorShuffleAsTruncBroadcast ( const SDLoc DL,
MVT  VT,
SDValue  V0,
int  BroadcastIdx,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsZeroOrAnyExtend ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to lower a vector shuffle as a zero extension on any microarch.

This routine will try to do everything in its power to cleverly lower a shuffle which happens to match the pattern of a zero extend. It doesn't check for the profitability of this lowering, it tries to aggressively match this pattern. It will use all of the micro-architectural details it can to emit an efficient lowering. It handles both blends with all-zero inputs to explicitly zero-extend and undef-lanes (sometimes undef due to masking out later).

The reason we have dedicated lowering for zext-style shuffles is that they are both incredibly common and often quite performance sensitive.

Definition at line 9159 of file X86ISelLowering.cpp.

References assert(), llvm::tgtok::Bits, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), i, isSequentialOrUndefInRange(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), llvm::ArrayRef< T >::size(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, and llvm::X86ISD::VZEXT_MOVL.

Referenced by lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().

static SDValue lowerVectorShuffleByMerging128BitLanes ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.

This will only succeed when the result of fixing the 128-bit lanes results in a single-input non-lane-crossing shuffle with a repeating shuffle mask in each 128-bit lanes. This handles many cases where we can quickly blend away the lane crosses early and then use simpler shuffles within each lane.

FIXME: It might be worthwhile at some point to support this without requiring the 128-bit lane-relative shuffles to be repeating, but currently in x86 only floating point has interesting non-repeating shuffles, and even those are still marginally more expensive.

Definition at line 11719 of file X86ISelLowering.cpp.

References assert(), llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), i, llvm::MVT::i64, is128BitLaneCrossingShuffleMask(), llvm::MVT::isFloatingPoint(), llvm::SDValue::isUndef(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().

static SDValue lowerVectorShuffleToEXPAND ( const SDLoc DL,
MVT  VT,
const SmallBitVector Zeroable,
ArrayRef< int >  Mask,
SDValue V1,
SDValue V2,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue lowerVectorShuffleWithPERMV ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleWithPSHUFB ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
const SmallBitVector Zeroable,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleWithSHUFPD ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleWithSHUFPS ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG 
)
static

Lower a vector shuffle using the SHUFPS instruction.

This is a helper routine dedicated to lowering vector shuffles using SHUFPS. It makes no assumptions about whether this is the best lowering, it simply uses it.

Definition at line 10093 of file X86ISelLowering.cpp.

References llvm::ArrayRef< T >::begin(), llvm::count_if(), llvm::find_if(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), llvm::X86ISD::SHUFP, std::swap(), and llvm::NVPTX::PTXLdStInstCode::V2.

Referenced by lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV4F32VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().

static SDValue lowerVectorShuffleWithSSE4A ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const SmallBitVector Zeroable,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleWithUndefHalf ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleWithUNPCK ( const SDLoc DL,
MVT  VT,
ArrayRef< int >  Mask,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG 
)
static
static SDValue lowerVSELECTtoVectorShuffle ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVSETCC ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 16679 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), ChangeVSETULTtoVSETULE(), llvm::X86ISD::CMPM, llvm::X86ISD::CMPP, EQ, llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::APInt::getSignBit(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::AArch64CC::GT, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE42(), llvm::X86Subtarget::hasVLX(), llvm::X86Subtarget::hasXOP(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), llvm::ISD::isUnsignedIntSetCC(), llvm_unreachable, Lower256IntVSETCC(), LowerBoolVSETCC_AVX512(), LowerIntVSETCC_AVX512(), llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, SB, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::X86ISD::SUBUS, std::swap(), translateX86FSETCC(), llvm::ISD::TRUNCATE, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::X86ISD::VPCOM, llvm::X86ISD::VPCOMU, llvm::ISD::XOR, and Zero.

Referenced by combineSetCC().

static SDValue lowerX86CmpEqZeroToCtlzSrl ( SDValue  Op,
EVT  ExtTy,
SelectionDAG DAG 
)
static
static SDValue lowerX86FPLogicOp ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue LowerXALUO ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerZERO_EXTEND ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerZERO_EXTEND_AVX512 ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue MarkEHGuard ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue MarkEHRegistrationNode ( SDValue  Op,
SelectionDAG DAG 
)
static
static bool matchAsm ( StringRef  S,
ArrayRef< const char * >  Pieces 
)
static
static bool matchBinaryPermuteVectorShuffle ( MVT  MaskVT,
ArrayRef< int >  Mask,
bool  FloatDomain,
SDValue V1,
SDValue V2,
SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget,
unsigned Shuffle,
MVT ShuffleVT,
unsigned PermuteImm 
)
static
static bool matchBinaryVectorShuffle ( MVT  MaskVT,
ArrayRef< int >  Mask,
bool  FloatDomain,
SDValue V1,
SDValue V2,
const X86Subtarget Subtarget,
unsigned Shuffle,
MVT ShuffleVT,
bool  IsUnary 
)
static
static SDValue matchBinOpReduction ( SDNode Extract,
ISD::NodeType  BinOp 
)
static
static bool MatchingStackOffset ( SDValue  Arg,
unsigned  Offset,
ISD::ArgFlagsTy  Flags,
MachineFrameInfo MFI,
const MachineRegisterInfo MRI,
const X86InstrInfo TII,
const CCValAssign VA 
)
static
static bool matchUnaryPermuteVectorShuffle ( MVT  MaskVT,
ArrayRef< int >  Mask,
bool  FloatDomain,
const X86Subtarget Subtarget,
unsigned Shuffle,
MVT ShuffleVT,
unsigned PermuteImm 
)
static
static bool matchUnaryVectorShuffle ( MVT  MaskVT,
ArrayRef< int >  Mask,
bool  FloatDomain,
const X86Subtarget Subtarget,
unsigned Shuffle,
MVT SrcVT,
MVT DstVT 
)
static
static int matchVectorShuffleAsByteRotate ( MVT  VT,
SDValue V1,
SDValue V2,
ArrayRef< int >  Mask 
)
static

Try to lower a vector shuffle as a byte rotation.

SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary byte-rotation of the concatenation of two vectors; pre-SSSE3 can use a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will try to generically lower a vector shuffle through such an pattern. It does not check for the profitability of lowering either as PALIGNR or PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form. This matches shuffle vectors that look like:

v8i16 [11, 12, 13, 14, 15, 0, 1, 2]

Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.

Definition at line 8635 of file X86ISelLowering.cpp.

References llvm::any_of(), is128BitLaneRepeatedShuffleMask(), matchVectorShuffleAsRotate(), llvm::SmallVectorTemplateCommon< T >::size(), and llvm::SM_SentinelZero.

Referenced by lowerVectorShuffleAsByteRotate(), and matchBinaryPermuteVectorShuffle().

static bool matchVectorShuffleAsInsertPS ( SDValue V1,
SDValue V2,
unsigned InsertPSMask,
const SmallBitVector Zeroable,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static
static int matchVectorShuffleAsRotate ( SDValue V1,
SDValue V2,
ArrayRef< int >  Mask 
)
static

Try to lower a vector shuffle as a rotation.

This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.

Definition at line 8550 of file X86ISelLowering.cpp.

References assert(), llvm::MipsISD::Hi, i, llvm::MipsISD::Lo, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::NVPTX::PTXLdStInstCode::V2.

Referenced by lowerVectorShuffleAsRotate(), and matchVectorShuffleAsByteRotate().

static int matchVectorShuffleAsShift ( MVT ShiftVT,
unsigned Opcode,
unsigned  ScalarSizeInBits,
ArrayRef< int >  Mask,
int  MaskOffset,
const SmallBitVector Zeroable,
const X86Subtarget Subtarget 
)
static

Try to lower a vector shuffle as a bit shift (shifts in zeros).

Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function matches elements from one of the input vectors shuffled to the left or right with zeroable elements 'shifted in'. It handles both the strictly bit-wise element shifts and the byte shift across an entire 128-bit double quad word lane.

PSHL : (little-endian) left bit shift. [ zz, 0, zz, 2 ] [ -1, 4, zz, -1 ] PSRL : (little-endian) right bit shift. [ 1, zz, 3, zz] [ -1, -1, 7, zz] PSLLDQ : (little-endian) left byte shift [ zz, 0, 1, 2, 3, 4, 5, 6] [ zz, zz, -1, -1, 2, 3, 4, -1] [ zz, zz, zz, zz, zz, zz, -1, 1] PSRLDQ : (little-endian) right byte shift [ 5, 6, 7, zz, zz, zz, zz, zz] [ -1, 5, 6, 7, zz, zz, zz, zz] [ 1, 2, -1, -1, -1, -1, zz, zz]

Definition at line 8757 of file X86ISelLowering.cpp.

References llvm::MVT::getIntegerVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), i, llvm::MVT::i8, isSequentialOrUndefInRange(), llvm::Left, llvm::ArrayRef< T >::size(), llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VSRLI.

Referenced by lowerVectorShuffleAsShift(), and matchUnaryPermuteVectorShuffle().

static bool matchVectorShuffleWithSHUFPD ( MVT  VT,
SDValue V1,
SDValue V2,
unsigned ShuffleImm,
ArrayRef< int >  Mask 
)
static
static SDValue MaterializeSETB ( const SDLoc DL,
SDValue  EFLAGS,
SelectionDAG DAG,
MVT  VT 
)
static
static SDValue materializeVectorConstant ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Create a vector constant without a load.

SSE/AVX provide the bare minimum functionality to do this, so it's all zeros, all ones, or some derivation that is cheap to calculate.

Definition at line 7327 of file X86ISelLowering.cpp.

References llvm::SDValue::getNode(), getOnesVector(), llvm::SDValue::getSimpleValueType(), getZeroVector(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::v16i32, llvm::MVT::v4i32, and llvm::MVT::v8i32.

static bool MayFoldIntoStore ( SDValue  Op)
static
static bool MayFoldIntoZeroExtend ( SDValue  Op)
static
static bool MayFoldLoad ( SDValue  Op)
static
static bool mayTailCallThisCC ( CallingConv::ID  CC)
static
static SDValue OptimizeConditionalInDecrement ( SDNode N,
SelectionDAG DAG 
)
static
static void Passv64i1ArgInRegs ( const SDLoc Dl,
SelectionDAG DAG,
SDValue  Chain,
SDValue Arg,
SmallVector< std::pair< unsigned, SDValue >, 8 > &  RegsToPass,
CCValAssign VA,
CCValAssign NextVA,
const X86Subtarget Subtarget 
)
static
static SDValue peekThroughBitcasts ( SDValue  V)
static
static SDValue peekThroughOneUseBitcasts ( SDValue  V)
static
static SDValue performShiftToAllZeros ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Returns a vector of 0s if the node in input is a vector logical shift by a constant amount which is known to be bigger than or equal to the vector element size in bits.

Definition at line 30352 of file X86ISelLowering.cpp.

References llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SDNode::getValueType(), getZeroVector(), llvm::X86Subtarget::hasInt256(), llvm::APInt::trunc(), llvm::APInt::uge(), llvm::MVT::v16i16, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8i16, and llvm::MVT::v8i32.

Referenced by combineShift().

static SDValue promoteExtBeforeAdd ( SDNode Ext,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

sext(add_nsw(x, C)) –> add(sext(x), C_sext) zext(add_nuw(x, C)) –> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode.

This can eliminate extend, add, and shift instructions.

Definition at line 32865 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::dyn_cast(), fuzzer::Flags, llvm::SelectionDAG::getConstant(), llvm::SDNode::getFlags(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDNodeFlags::hasNoSignedWrap(), llvm::SDNodeFlags::hasNoUnsignedWrap(), llvm::MVT::i64, llvm::SDNodeFlags::setNoSignedWrap(), llvm::SDNodeFlags::setNoUnsignedWrap(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SDNode::uses(), and llvm::ISD::ZERO_EXTEND.

Referenced by combineSext(), and combineZext().

static SDValue recoverFramePointer ( SelectionDAG DAG,
const Function Fn,
SDValue  EntryEBP 
)
static

When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP.

Here's the math: RegNodeBase = EntryEBP - RegNodeSize ParentFP = RegNodeBase - ParentFrameOffset Subtracting RegNodeSize takes us to the offset of the registration node, and subtracting the offset (negative on x86) takes us back to the parent FP.

Definition at line 18747 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::SelectionDAG::getConstant(), llvm::MachineModuleInfo::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMCSymbol(), llvm::MachineFunction::getMMI(), llvm::Value::getName(), llvm::SelectionDAG::getNode(), llvm::MCContext::getOrCreateParentFrameOffsetSymbol(), llvm::TargetLoweringBase::getPointerTy(), llvm::GlobalValue::getRealLinkageName(), getSEHRegistrationNodeSize(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasPersonalityFn(), llvm::X86Subtarget::is64Bit(), llvm::ISD::LOCAL_RECOVER, and llvm::ISD::SUB.

Referenced by LowerINTRINSIC_WO_CHAIN().

static SDValue reduceMaskedLoadToScalarLoad ( MaskedLoadSDNode ML,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert.

Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.

Definition at line 31508 of file X86ISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::MemSDNode::getPointerInfo(), llvm::MaskedLoadSDNode::getSrc0(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_VECTOR_ELT, and llvm::SPII::Load.

Referenced by combineMaskedLoad().

static SDValue reduceMaskedStoreToScalarStore ( MaskedStoreSDNode MS,
SelectionDAG DAG 
)
static

If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store.

Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.

Definition at line 31675 of file X86ISelLowering.cpp.

References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::MemSDNode::getPointerInfo(), llvm::SelectionDAG::getStore(), llvm::MaskedStoreSDNode::getValue(), llvm::SDValue::getValueType(), and llvm::EVT::getVectorElementType().

Referenced by combineMaskedStore().

static SDValue reduceVMULWidth ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code.

Two typical patterns are handled: Pattern1: %2 = sext/zext <N x="" i8>=""> %1 to <N x="" i32>=""> %4 = sext/zext <N x="" i8>=""> %3 to <N x="" i32>=""> %5 = mul <N x="" i32>=""> %2, %4

Pattern2: %2 = zext/sext <N x="" i16>=""> %1 to <N x="" i32>=""> %4 = zext/sext <N x="" i16>=""> %3 to <N x="" i32>=""> or %4 = build_vector <N x="" i32>=""> C1, ..., CN (C1..CN are constants) %5 = mul <N x="" i32>=""> %2, %4

There are four mul shrinking modes: If %2 == sext32(trunc8(%2)), i.e., the scalar value range of %2 is -128 to 128, and the scalar value range of %4 is also -128 to 128, generate pmullw+sext32 for it (MULS8 mode). If %2 == zext32(trunc8(%2)), i.e., the scalar value range of %2 is 0 to 255, and the scalar value range of %4 is also 0 to 255, generate pmullw+zext32 for it (MULU8 mode). If %2 == sext32(trunc16(%2)), i.e., the scalar value range of %2 is -32768 to 32767, and the scalar value range of %4 is also -32768 to 32767, generate pmullw+pmulhw for it (MULS16 mode). If %2 == zext32(trunc16(%2)), i.e., the scalar value range of %2 is 0 to 65535, and the scalar value range of %4 is also 0 to 65535, generate pmullw+pmulhuw for it (MULU16 mode).

Definition at line 30027 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, canReduceVMulWidth(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), i, llvm::MVT::i16, llvm::MVT::i32, llvm::X86Subtarget::isPMULLDSlow(), Mode, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, MULS16, MULS8, MULU8, llvm::Function::optForMinSize(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::TRUNCATE, llvm::X86ISD::UNPCKL, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by combineMul().

static bool resolveTargetShuffleInputs ( SDValue  Op,
SDValue Op0,
SDValue Op1,
SmallVectorImpl< int > &  Mask 
)
static

Calls setTargetShuffleZeroElements to resolve a target shuffle mask's inputs and set the SM_SentinelUndef and SM_SentinelZero values.

Then check the remaining input indices in case we now have a unary shuffle and adjust the Op0/Op1 inputs accordingly. Returns true if the target shuffle mask was decoded.

Definition at line 5818 of file X86ISelLowering.cpp.

References llvm::any_of(), getFauxShuffleMask(), setTargetShuffleZeroElements(), and llvm::SmallVectorTemplateCommon< T, typename >::size().

Referenced by combineX86ShufflesRecursively().

static void scaleShuffleMask ( int  Scale,
ArrayRef< int >  Mask,
SmallVectorImpl< int > &  ScaledMask 
)
static

Helper function to scale a shuffle or target shuffle mask, replacing each mask index with the scaled sequential indices for an equivalent narrowed mask.

This is the reverse process to canWidenShuffleElements, but can always succeed.

Definition at line 4607 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), i, and llvm::ArrayRef< T >::size().

Referenced by combineX86ShuffleChain(), lowerV4I64VectorShuffle(), lowerV8I64VectorShuffle(), and matchUnaryPermuteVectorShuffle().

static bool setTargetShuffleZeroElements ( SDValue  N,
SmallVectorImpl< int > &  Mask,
SmallVectorImpl< SDValue > &  Ops 
)
static
static bool shouldGuaranteeTCO ( CallingConv::ID  CC,
bool  GuaranteedTailCallOpt 
)
static

Return true if the function is being made into a tailcall target by changing its ABI.

Definition at line 2642 of file X86ISelLowering.cpp.

References canGuaranteeTCO().

Referenced by llvm::X86::isCalleePop().

static SDValue splitAndLowerVectorShuffle ( const SDLoc DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int >  Mask,
SelectionDAG DAG 
)
static
STATISTIC ( NumTailCalls  ,
"Number of tail calls  
)
static bool SupportedVectorShiftWithBaseAmnt ( MVT  VT,
const X86Subtarget Subtarget,
unsigned  Opcode 
)
static

Definition at line 21287 of file X86ISelLowering.cpp.

References SupportedVectorShiftWithImm().

Referenced by LowerScalarVariableShift().

static bool SupportedVectorShiftWithImm ( MVT  VT,
const X86Subtarget Subtarget,
unsigned  Opcode 
)
static
static bool SupportedVectorVarShift ( MVT  VT,
const X86Subtarget Subtarget,
unsigned  Opcode 
)
static
static X86::CondCode TranslateIntegerX86CC ( ISD::CondCode  SetCCOpcode)
static
static X86::CondCode TranslateX86CC ( ISD::CondCode  SetCCOpcode,
const SDLoc DL,
bool  isFP,
SDValue LHS,
SDValue RHS,
SelectionDAG DAG 
)
static
static int translateX86FSETCC ( ISD::CondCode  SetCCOpcode,
SDValue Op0,
SDValue Op1 
)
static
static SDValue truncateVectorCompareWithPACKSS ( EVT  DstVT,
SDValue  In,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue WidenMaskArithmetic ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue XFormVExtractWithShuffleIntoLoad ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

Check if a vector extract from a target-specific shuffle of a load can be folded into a single element load.

Similar handling for VECTOR_SHUFFLE is performed by DAGCombiner, but shuffles have been custom lowered so we need to handle those here.

Definition at line 28292 of file X86ISelLowering.cpp.

References assert(), llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::DataLayout::getABITypeAlignment(), llvm::MemSDNode::getAlignment(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::EVT::getTypeForEVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), isTargetShuffle(), llvm::EVT::isVector(), llvm::MemSDNode::isVolatile(), llvm::ISD::LOAD, llvm::SmallVectorTemplateCommon< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineExtractVectorElt().

Variable Documentation

cl::opt<bool> ExperimentalVectorWideningLegalization("x86-experimental-vector-widening-legalization", cl::init(false), cl::desc("Enable an experimental vector type legalization through widening ""rather than promotion."), cl::Hidden)
static