LLVM  3.7.0
Macros | Enumerations | Functions | Variables
X86ISelLowering.cpp File Reference
#include "X86ISelLowering.h"
#include "Utils/X86ShuffleDecode.h"
#include "X86CallingConv.h"
#include "X86FrameLowering.h"
#include "X86InstrBuilder.h"
#include "X86MachineFunctionInfo.h"
#include "X86TargetMachine.h"
#include "X86TargetObjectFile.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/CodeGen/IntrinsicLowering.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/CallSite.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
#include "X86IntrinsicsInfo.h"
#include <bitset>
#include <numeric>
#include <cctype>
#include "X86GenCallingConv.inc"
Include dependency graph for X86ISelLowering.cpp:

Go to the source code of this file.

Macros

#define DEBUG_TYPE   "x86-isel"
 

Enumerations

enum  StructReturnType { NotStructReturn, RegStructReturn, StackStructReturn }
 CallIsStructReturn - Determines whether a call uses struct return semantics. More...
 

Functions

 STATISTIC (NumTailCalls,"Number of tail calls")
 
static SDValue getMOVL (SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1, SDValue V2)
 getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width. More...
 
static void getMaxByValAlign (Type *Ty, unsigned &MaxAlign)
 Helper for getByValTypeAlignment to determine the desired ByVal argument alignment. More...
 
static StructReturnType callIsStructReturn (const SmallVectorImpl< ISD::OutputArg > &Outs)
 
static StructReturnType argsAreStructReturn (const SmallVectorImpl< ISD::InputArg > &Ins)
 Determines whether a function uses struct return semantics. More...
 
static SDValue CreateCopyOfByValArgument (SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, SDLoc dl)
 Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute. More...
 
static bool IsTailCallConvention (CallingConv::ID CC)
 Return true if the calling convention is one that supports tail call optimization. More...
 
static bool IsCCallConvention (CallingConv::ID CC)
 Return true if the calling convention is a C calling convention. More...
 
static bool FuncIsMadeTailCallSafe (CallingConv::ID CC, bool GuaranteedTailCallOpt)
 Return true if the function is being made into a tailcall target by changing its ABI. More...
 
static ArrayRef< MCPhysRegget64BitArgumentGPRs (CallingConv::ID CallConv, const X86Subtarget *Subtarget)
 
static ArrayRef< MCPhysRegget64BitArgumentXMMs (MachineFunction &MF, CallingConv::ID CallConv, const X86Subtarget *Subtarget)
 
static SDValue EmitTailCallStoreRetAddr (SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue RetAddrFrIdx, EVT PtrVT, unsigned SlotSize, int FPDiff, SDLoc dl)
 Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0). More...
 
static bool MatchingStackOffset (SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo *MFI, const MachineRegisterInfo *MRI, const X86InstrInfo *TII)
 MatchingStackOffset - Return true if the given stack call argument is already available in the same position (relatively) of the caller's incoming argument stack. More...
 
static bool MayFoldLoad (SDValue Op)
 
static bool MayFoldIntoStore (SDValue Op)
 
static bool isTargetShuffle (unsigned Opcode)
 
static SDValue getTargetShuffleNode (unsigned Opc, SDLoc dl, EVT VT, SDValue V1, unsigned TargetMask, SelectionDAG &DAG)
 
static SDValue getTargetShuffleNode (unsigned Opc, SDLoc dl, EVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
 
static bool isX86CCUnsigned (unsigned X86CC)
 Return true if the condition is an unsigned comparison operation. More...
 
static unsigned TranslateX86CC (ISD::CondCode SetCCOpcode, SDLoc DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG)
 TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86 specific condition code, returning the condition code and the LHS/RHS of the comparison to make. More...
 
static bool hasFPCMov (unsigned X86CC)
 hasFPCMov - is there a floating point cmov for the specific X86 condition code. More...
 
static bool isUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
 isUndefInRange - Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef. More...
 
static bool isUndefOrInRange (int Val, int Low, int Hi)
 isUndefOrInRange - Return true if Val is undef or if its value falls within the specified range (L, H]. More...
 
static bool isUndefOrEqual (int Val, int CmpVal)
 isUndefOrEqual - Val is either less than zero (undef) or equal to the specified value. More...
 
static bool isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low)
 isSequentialOrUndefInRange - Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size]. More...
 
static bool isVEXTRACTIndex (SDNode *N, unsigned vecWidth)
 isVEXTRACTIndex - Return true if the specified EXTRACT_SUBVECTOR operand specifies a vector extract that is suitable for instruction that extract 128 or 256 bit vectors More...
 
static bool isVINSERTIndex (SDNode *N, unsigned vecWidth)
 isVINSERTIndex - Return true if the specified INSERT_SUBVECTOR operand specifies a subvector insert that is suitable for input to insertion of 128 or 256-bit subvectors More...
 
static unsigned getExtractVEXTRACTImmediate (SDNode *N, unsigned vecWidth)
 
static unsigned getInsertVINSERTImmediate (SDNode *N, unsigned vecWidth)
 
static bool isZero (SDValue V)
 isZero - Returns true if Elt is a constant integer zero More...
 
static SDValue getZeroVector (EVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG, SDLoc dl)
 getZeroVector - Returns a vector of specified type with all zero elements. More...
 
static SDValue ExtractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl, unsigned vectorWidth)
 
static SDValue Extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl)
 Generate a DAG to grab 128-bits from a vector > 128 bits. More...
 
static SDValue Extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl)
 Generate a DAG to grab 256-bits from a 512-bit vector. More...
 
static SDValue InsertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl, unsigned vectorWidth)
 
static SDValue Insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl)
 Generate a DAG to put 128-bits into a vector > 128 bits. More...
 
static SDValue Insert256BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl)
 
static SDValue Concat128BitVectors (SDValue V1, SDValue V2, EVT VT, unsigned NumElems, SelectionDAG &DAG, SDLoc dl)
 Concat two 128-bit vectors into a 256 bit vector using VINSERTF128 instructions. More...
 
static SDValue Concat256BitVectors (SDValue V1, SDValue V2, EVT VT, unsigned NumElems, SelectionDAG &DAG, SDLoc dl)
 
static SDValue getOnesVector (MVT VT, bool HasInt256, SelectionDAG &DAG, SDLoc dl)
 getOnesVector - Returns a vector of specified type with all bits set. More...
 
static SDValue getUnpackl (SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, SDValue V2)
 getUnpackl - Returns a vector_shuffle node for an unpackl operation. More...
 
static SDValue getUnpackh (SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, SDValue V2)
 getUnpackh - Returns a vector_shuffle node for an unpackh operation. More...
 
static SDValue getShuffleVectorZeroOrUndef (SDValue V2, unsigned Idx, bool IsZero, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified vector of zero or undef vector. More...
 
static bool getTargetShuffleMask (SDNode *N, MVT VT, SmallVectorImpl< int > &Mask, bool &IsUnary)
 getTargetShuffleMask - Calculates the shuffle mask corresponding to the target specific opcode. More...
 
static SDValue getShuffleScalarElt (SDNode *N, unsigned Index, SelectionDAG &DAG, unsigned Depth)
 getShuffleScalarElt - Returns the scalar element that will make up the ith element of the result of the vector shuffle. More...
 
static SDValue LowerBuildVectorv16i8 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget *Subtarget, const TargetLowering &TLI)
 LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. More...
 
static SDValue LowerBuildVectorv8i16 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget *Subtarget, const TargetLowering &TLI)
 LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. More...
 
static SDValue LowerBuildVectorv4x32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget, const TargetLowering &TLI)
 LowerBuildVectorv4x32 - Custom lower build_vector of v4i32 or v4f32. More...
 
static SDValue getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, SDLoc dl)
 Return a vector logical shift node. More...
 
static SDValue LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, SDLoc dl, SelectionDAG &DAG)
 
static SDValue EltsFromConsecutiveLoads (EVT VT, ArrayRef< SDValue > Elts, SDLoc &DL, SelectionDAG &DAG, bool isAfterLegalize)
 Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'. More...
 
static SDValue LowerVectorBroadcast (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 LowerVectorBroadcast - Attempt to use the vbroadcast instruction to generate a splat value for the following cases: More...
 
static int getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx)
 For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index. More...
 
static SDValue buildFromShuffleMostly (SDValue Op, SelectionDAG &DAG)
 
static SDValue ConvertI1VectorToInterger (SDValue Op, SelectionDAG &DAG)
 
static bool isHorizontalBinOp (const BuildVectorSDNode *N, unsigned Opcode, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1)
 Return true if N implements a horizontal binop and return the operands for the horizontal binop into V0 and V1. More...
 
static SDValue ExpandHorizontalBinOp (const SDValue &V0, const SDValue &V1, SDLoc DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI)
 Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector. More...
 
static SDValue LowerToAddSub (const BuildVectorSDNode *BV, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Try to fold a build_vector that performs an 'addsub' to an X86ISD::ADDSUB node. More...
 
static SDValue LowerToHorizontalOp (const BuildVectorSDNode *BV, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Lower BUILD_VECTOR to a horizontal add/sub operation if possible. More...
 
static SDValue LowerAVXCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerCONCAT_VECTORSvXi1 (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCONCAT_VECTORS (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static bool isNoopShuffleMask (ArrayRef< int > Mask)
 Tiny helper function to identify a no-op mask. More...
 
static bool isSingleInputShuffleMask (ArrayRef< int > Mask)
 Helper function to classify a mask as a single-input mask. More...
 
static bool is128BitLaneCrossingShuffleMask (MVT VT, ArrayRef< int > Mask)
 Test whether there are elements crossing 128-bit lanes in this shuffle mask. More...
 
static bool is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each 128-bit lane. More...
 
static bool isShuffleEquivalent (SDValue V1, SDValue V2, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask)
 Checks whether a shuffle mask is equivalent to an explicit list of arguments. More...
 
static SDValue getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, SDLoc DL, SelectionDAG &DAG)
 Get a 4-lane 8-bit shuffle immediate for a mask. More...
 
static SDValue lowerVectorShuffleAsBitBlend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to emit a blend instruction for a shuffle using bit math. More...
 
static SDValue lowerVectorShuffleAsBlend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Try to emit a blend instruction for a shuffle. More...
 
static SDValue lowerVectorShuffleAsBlendAndPermute (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to lower as a blend of elements from two inputs followed by a single-input permutation. More...
 
static SDValue lowerVectorShuffleAsDecomposedShuffleBlend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Generic routine to decompose a shuffle and blend into indepndent blends and permutes. More...
 
static SDValue lowerVectorShuffleAsByteRotate (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a byte rotation. More...
 
static SmallBitVector computeZeroableShuffleElements (ArrayRef< int > Mask, SDValue V1, SDValue V2)
 Compute whether each element of a shuffle is zeroable. More...
 
static SDValue lowerVectorShuffleAsBitMask (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to emit a bitmask instruction for a shuffle. More...
 
static SDValue lowerVectorShuffleAsShift (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to lower a vector shuffle as a bit shift (shifts in zeros). More...
 
static SDValue lowerVectorShuffleWithSSE4A (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ. More...
 
static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend (SDLoc DL, MVT VT, int Scale, bool AnyExt, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Lower a vector shuffle as a zero or any extension. More...
 
static SDValue lowerVectorShuffleAsZeroOrAnyExtend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a zero extension on any microarch. More...
 
static SDValue getScalarValueForVectorElement (SDValue V, int Idx, SelectionDAG &DAG)
 Try to get a scalar value for a specific element of a vector. More...
 
static bool isShuffleFoldableLoad (SDValue V)
 Helper to test for a load that can be folded with x86 shuffles. More...
 
static SDValue lowerVectorShuffleAsElementInsertion (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Try to lower insertion of a single element into a zero vector. More...
 
static SDValue lowerVectorShuffleAsBroadcast (SDLoc DL, MVT VT, SDValue V, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Try to lower broadcast of a single element. More...
 
static SDValue lowerVectorShuffleAsInsertPS (SDValue Op, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 
static SDValue lowerVectorShuffleAsUnpack (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction. More...
 
static SDValue lowerV2F64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 2-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV2I64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 2-lane 64-bit integer shuffles. More...
 
static bool isSingleSHUFPSMask (ArrayRef< int > Mask)
 Test whether this can be lowered with a single SHUFPS instruction. More...
 
static SDValue lowerVectorShuffleWithSHUFPS (SDLoc DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 Lower a vector shuffle using the SHUFPS instruction. More...
 
static SDValue lowerV4F32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Lower 4-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV4I32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Lower 4-lane i32 vector shuffles. More...
 
static SDValue lowerV8I16GeneralSingleInputVectorShuffle (SDLoc DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part. More...
 
static SDValue lowerVectorShuffleAsPSHUFB (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse)
 Helper to form a PSHUFB-based shuffle+blend. More...
 
static SDValue lowerV8I16VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Generic lowering of 8-lane i16 shuffles. More...
 
static int canLowerByDroppingEvenElements (ArrayRef< int > Mask)
 Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped. More...
 
static SDValue lowerV16I8VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Generic lowering of v16i8 shuffles. More...
 
static SDValue lower128BitVectorShuffle (SDValue Op, SDValue V1, SDValue V2, MVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Dispatching routine to lower various 128-bit x86 vector shuffles. More...
 
static bool canWidenShuffleElements (ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask)
 Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled. More...
 
static SDValue splitAndLowerVectorShuffle (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Generic routine to split vector shuffle into half-sized shuffles. More...
 
static SDValue lowerVectorShuffleAsSplitOrBlend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Either split a vector in halves or decompose the shuffles and the blend. More...
 
static SDValue lowerVectorShuffleAsLanePermuteAndBlend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Lower a vector shuffle crossing multiple 128-bit lanes as a permutation and blend of those lanes. More...
 
static SDValue lowerV2X128VectorShuffle (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering 2-lane 128-bit shuffles. More...
 
static SDValue lowerVectorShuffleByMerging128BitLanes (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane. More...
 
static bool isShuffleMaskInputInPlace (int Input, ArrayRef< int > Mask)
 Test whether the specified input (0 or 1) is in-place blended by the given mask. More...
 
static SDValue lowerVectorShuffleWithSHUFPD (SDLoc DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 
static SDValue lowerV4F64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 4-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV4I64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 4-lane 64-bit integer shuffles. More...
 
static SDValue lowerV8F32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV8I32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 32-bit integer shuffles. More...
 
static SDValue lowerV16I16VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 16-bit integer shuffles. More...
 
static SDValue lowerV32I8VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 32-lane 8-bit integer shuffles. More...
 
static SDValue lower256BitVectorShuffle (SDValue Op, SDValue V1, SDValue V2, MVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 High-level routine to lower various 256-bit x86 vector shuffles. More...
 
static SDValue lowerV8F64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV16F32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV8I64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 64-bit integer shuffles. More...
 
static SDValue lowerV16I32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 32-bit integer shuffles. More...
 
static SDValue lowerV32I16VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 32-lane 16-bit integer shuffles. More...
 
static SDValue lowerV64I8VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Handle lowering of 64-lane 8-bit integer shuffles. More...
 
static SDValue lower512BitVectorShuffle (SDValue Op, SDValue V1, SDValue V2, MVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 High-level routine to lower various 512-bit x86 vector shuffles. More...
 
static SDValue lowerVectorShuffle (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Top-level lowering for x86 vector shuffles. More...
 
static bool BUILD_VECTORtoBlendMask (BuildVectorSDNode *BuildVector, unsigned &MaskValue)
 
static SDValue lowerVSELECTtoVectorShuffle (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Try to lower a VSELECT instruction to a vector shuffle. More...
 
static SDValue LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerSCALAR_TO_VECTOR (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue GetTLSADDR (SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LocalDynamic=false)
 
static SDValue LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
 
static SDValue LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
 
static SDValue LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool is64Bit)
 
static SDValue LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC)
 
static SDValue LowerShiftParts (SDValue Op, SelectionDAG &DAG)
 LowerShiftParts - Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount. More...
 
static SDValue lowerUINT_TO_FP_vXi32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerAVXExtend (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue LowerZERO_EXTEND_AVX512 (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerANY_EXTEND (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerZERO_EXTEND (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerFP_EXTEND (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFABSorFNEG (SDValue Op, SelectionDAG &DAG)
 The only differences between FABS and FNEG are the mask and the logic op. More...
 
static SDValue LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFGETSIGN (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerVectorAllZeroTest (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static bool hasNonFlagsUse (SDValue Op)
 return true if Op has a use that doesn't just read flags. More...
 
static bool isAllOnes (SDValue V)
 
static int translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1)
 
  • Turns an ISD::CondCode into a value suitable for SSE floating point mask CMPs.
More...
 
static SDValue Lower256IntVSETCC (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerBoolVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerIntVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue ChangeVSETULTtoVSETULE (SDLoc dl, SDValue Op1, SelectionDAG &DAG)
 Try to turn a VSETULT into a VSETULE by modifying its second operand Op1. More...
 
static SDValue LowerVSETCC (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static bool isX86LogicalCmp (SDValue Op)
 
static bool isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG)
 
static SDValue LowerSIGN_EXTEND_AVX512 (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerSIGN_EXTEND_VECTOR_INREG (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerSIGN_EXTEND (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerExtendedLoad (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static bool isAndOrOfSetCCs (SDValue Op, unsigned &Opc)
 
static bool isXor1OfSetCC (SDValue Op)
 
static SDValue LowerVACOPY (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue getTargetVShiftByConstNode (unsigned Opc, SDLoc dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG)
 
static SDValue getTargetVShiftNode (unsigned Opc, SDLoc dl, MVT VT, SDValue SrcOp, SDValue ShAmt, SelectionDAG &DAG)
 
static SDValue getVectorMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting for Mask when lowering masking intrinsics. More...
 
static SDValue getScalarMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Creates an SDNode for a predicated scalar operation. More...
 
static int getSEHRegistrationNodeSize (const Function *Fn)
 
static SDValue recoverFramePointer (SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP)
 When the 32-bit MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP. More...
 
static SDValue LowerINTRINSIC_WO_CHAIN (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue getGatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget *Subtarget)
 
static SDValue getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain)
 
static SDValue getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain)
 
static void getReadPerformanceCounter (SDNode *N, SDLoc DL, SelectionDAG &DAG, const X86Subtarget *Subtarget, SmallVectorImpl< SDValue > &Results)
 
static void getReadTimeStampCounter (SDNode *N, SDLoc DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget *Subtarget, SmallVectorImpl< SDValue > &Results)
 
static SDValue LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerSEHRESTOREFRAME (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerCTLZ (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerCTLZ_ZERO_UNDEF (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerCTTZ (SDValue Op, SelectionDAG &DAG)
 
static SDValue Lower256IntArith (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerADD (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerSUB (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerMUL (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMUL_LOHI (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static bool SupportedVectorShiftWithImm (MVT VT, const X86Subtarget *Subtarget, unsigned Opcode)
 
static bool SupportedVectorShiftWithBaseAmnt (MVT VT, const X86Subtarget *Subtarget, unsigned Opcode)
 
static bool SupportedVectorVarShift (MVT VT, const X86Subtarget *Subtarget, unsigned Opcode)
 
static SDValue LowerScalarImmediateShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue LowerScalarVariableShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue LowerShift (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerXALUO (SDValue Op, SelectionDAG &DAG)
 
static bool hasMFENCE (const X86Subtarget &Subtarget)
 
static SDValue LowerATOMIC_FENCE (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCMP_SWAP (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerBITCAST (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerHorizontalByteSum (SDValue V, MVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 Compute the horizontal sum of bytes in V for the elements of VT. More...
 
static SDValue LowerVectorCTPOPInRegLUT (SDValue Op, SDLoc DL, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerVectorCTPOPBitmath (SDValue Op, SDLoc DL, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerVectorCTPOP (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTPOP (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerLOAD_SUB (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerADDC_ADDE_SUBC_SUBE (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFSINCOS (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMSCATTER (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMGATHER (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG)
 
static MachineBasicBlockEmitXBegin (MachineInstr *MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII)
 Utility function to emit xbegin specifying the start of an RTM region. More...
 
static MachineBasicBlockEmitPCMPSTRM (MachineInstr *MI, MachineBasicBlock *BB, const TargetInstrInfo *TII)
 
static MachineBasicBlockEmitPCMPSTRI (MachineInstr *MI, MachineBasicBlock *BB, const TargetInstrInfo *TII)
 
static MachineBasicBlockEmitMonitor (MachineInstr *MI, MachineBasicBlock *BB, const X86Subtarget *Subtarget)
 
static bool checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
 
static bool isShuffleHigh128VectorInsertLow (ShuffleVectorSDNode *SVOp)
 isShuffleHigh128VectorInsertLow - Checks whether the shuffle node is the same as extracting the high 128-bit part of 256-bit vector and then inserting the result into the low part of a new 256-bit vector More...
 
static bool isShuffleLow128VectorInsertHigh (ShuffleVectorSDNode *SVOp)
 isShuffleLow128VectorInsertHigh - Checks whether the shuffle node is the same as extracting the low 128-bit part of 256-bit vector and then inserting the result into the high part of a new 256-bit vector More...
 
static SDValue PerformShuffleCombine256 (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 PerformShuffleCombine256 - Performs shuffle combines for 256-bit vectors. More...
 
static bool combineX86ShuffleChain (SDValue Op, SDValue Root, ArrayRef< int > Mask, int Depth, bool HasPSHUFB, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 Combine an arbitrary chain of shuffles into a single instruction if possible. More...
 
static bool combineX86ShufflesRecursively (SDValue Op, SDValue Root, ArrayRef< int > RootMask, int Depth, bool HasPSHUFB, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 Fully generic combining of x86 shuffle instructions. More...
 
static SmallVector< int, 4 > getPSHUFShuffleMask (SDValue N)
 Get the PSHUF-style mask from PSHUF node. More...
 
static SDValue combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 Search for a combinable shuffle across a chain ending in pshufd. More...
 
static bool combineRedundantHalfShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 Search for a combinable shuffle across a chain ending in pshuflw or pshufhw. More...
 
static SDValue PerformTargetShuffleCombine (SDValue N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 Try to combine x86 target specific shuffles. More...
 
static SDValue combineShuffleToAddSub (SDNode *N, SelectionDAG &DAG)
 Try to combine a shuffle into a target-specific add-sub node. More...
 
static SDValue PerformShuffleCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 PerformShuffleCombine - Performs several different shuffle combines. More...
 
static SDValue XFormVExtractWithShuffleIntoLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 XFormVExtractWithShuffleIntoLoad - Check if a vector extract from a target specific shuffle of a load can be folded into a single element load. More...
 
static SDValue PerformBITCASTCombine (SDNode *N, SelectionDAG &DAG)
 Detect bitcasts between i32 to x86mmx low word. More...
 
static SDValue PerformEXTRACT_VECTOR_ELTCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 PerformEXTRACT_VECTOR_ELTCombine - Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence. More...
 
static std::pair< unsigned, boolmatchIntegerMINMAX (SDValue Cond, EVT VT, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 Matches a VSELECT onto min/max or return 0 if the node doesn't match. More...
 
static SDValue transformVSELECTtoBlendVECTOR_SHUFFLE (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue PerformSELECTCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 PerformSELECTCombine - Do target-specific dag combines on SELECT and VSELECT nodes. More...
 
static SDValue checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC)
 
static bool checkBoolTestAndOrSetCCCombine (SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd)
 Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS. More...
 
static SDValue PerformCMOVCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]. More...
 
static SDValue PerformINTRINSIC_WO_CHAINCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue PerformMulCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 PerformMulCombine - Optimize a single multiply with constant into two in order to implement it with two cheaper instructions, e.g. More...
 
static SDValue PerformSHLCombine (SDNode *N, SelectionDAG &DAG)
 
static SDValue performShiftToAllZeros (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 Returns a vector of 0s if the node in input is a vector logical shift by a constant amount which is known to be bigger than or equal to the vector element size in bits. More...
 
static SDValue PerformShiftCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 PerformShiftCombine - Combine shifts. More...
 
static SDValue CMPEQCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 
static bool CanFoldXORWithAllOnes (const SDNode *N)
 CanFoldXORWithAllOnes - Test whether the XOR operand is a AllOnes vector so it can be folded inside ANDNP. More...
 
static SDValue WidenMaskArithmetic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 
static SDValue VectorZextCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 
static SDValue PerformAndCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 
static SDValue PerformOrCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 
static SDValue performIntegerAbsCombine (SDNode *N, SelectionDAG &DAG)
 
static SDValue PerformXorCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 
static SDValue PerformLOADCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 PerformLOADCombine - Do target-specific dag combines on LOAD nodes. More...
 
static SDValue PerformMLOADCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 PerformMLOADCombine - Resolve extending loads. More...
 
static SDValue PerformMSTORECombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 PerformMSTORECombine - Resolve truncating stores. More...
 
static SDValue PerformSTORECombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 PerformSTORECombine - Do target-specific dag combines on STORE nodes. More...
 
static bool isHorizontalBinOp (SDValue &LHS, SDValue &RHS, bool IsCommutative)
 Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS. More...
 
static SDValue PerformFADDCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 Do target-specific dag combines on floating point adds. More...
 
static SDValue PerformFSUBCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 Do target-specific dag combines on floating point subs. More...
 
static SDValue PerformFORCombine (SDNode *N, SelectionDAG &DAG)
 Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes. More...
 
static SDValue PerformFMinFMaxCombine (SDNode *N, SelectionDAG &DAG)
 Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes. More...
 
static SDValue PerformFANDCombine (SDNode *N, SelectionDAG &DAG)
 Do target-specific dag combines on X86ISD::FAND nodes. More...
 
static SDValue PerformFANDNCombine (SDNode *N, SelectionDAG &DAG)
 Do target-specific dag combines on X86ISD::FANDN nodes. More...
 
static SDValue PerformBTCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue PerformVZEXT_MOVLCombine (SDNode *N, SelectionDAG &DAG)
 
static SDValue PerformSIGN_EXTEND_INREGCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue PerformSExtCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 
static SDValue PerformFMACombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue PerformZExtCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 
static SDValue PerformISDSETCCCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue NarrowVectorLoadToElement (LoadSDNode *Load, unsigned Index, SelectionDAG &DAG)
 
static SDValue PerformINSERTPSCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue PerformBLENDICombine (SDNode *N, SelectionDAG &DAG)
 
static SDValue MaterializeSETB (SDLoc DL, SDValue EFLAGS, SelectionDAG &DAG, MVT VT)
 
static SDValue PerformSETCCCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 
static SDValue PerformBrCondCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 
static SDValue performVectorCompareAndMaskUnaryOpCombine (SDNode *N, SelectionDAG &DAG)
 
static SDValue PerformUINT_TO_FPCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue PerformSINT_TO_FPCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue PerformADCCombine (SDNode *N, SelectionDAG &DAG, X86TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue OptimizeConditionalInDecrement (SDNode *N, SelectionDAG &DAG)
 
static SDValue PerformAddCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 PerformADDCombine - Do target-specific dag combines on integer adds. More...
 
static SDValue PerformSubCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget)
 
static SDValue performVZEXTCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget)
 performVZEXTCombine - Performs build vector combines More...
 
static bool matchAsm (StringRef S, ArrayRef< const char * > Pieces)
 
static bool clobbersFlagRegisters (const SmallVector< StringRef, 4 > &AsmPieces)
 

Variables

static cl::opt< boolExperimentalVectorWideningLegalization ("x86-experimental-vector-widening-legalization", cl::init(false), cl::desc("Enable an experimental vector type legalization through widening ""rather than promotion."), cl::Hidden)
 

Macro Definition Documentation

#define DEBUG_TYPE   "x86-isel"

Definition at line 60 of file X86ISelLowering.cpp.

Enumeration Type Documentation

CallIsStructReturn - Determines whether a call uses struct return semantics.

Enumerator
NotStructReturn 
RegStructReturn 
StackStructReturn 

Definition at line 2241 of file X86ISelLowering.cpp.

Function Documentation

static StructReturnType argsAreStructReturn ( const SmallVectorImpl< ISD::InputArg > &  Ins)
static

Determines whether a function uses struct return semantics.

Definition at line 2261 of file X86ISelLowering.cpp.

References llvm::SmallVectorBase::empty(), llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSRet(), NotStructReturn, RegStructReturn, and StackStructReturn.

static bool BUILD_VECTORtoBlendMask ( BuildVectorSDNode BuildVector,
unsigned MaskValue 
)
static
static SDValue buildFromShuffleMostly ( SDValue  Op,
SelectionDAG DAG 
)
static
static StructReturnType callIsStructReturn ( const SmallVectorImpl< ISD::OutputArg > &  Outs)
static
static bool CanFoldXORWithAllOnes ( const SDNode N)
static
static int canLowerByDroppingEvenElements ( ArrayRef< int Mask)
static

Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped.

This handles shuffles which take every Nth element where N is a power of two. Example shuffle masks:

N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24

Any of these lanes can of course be undef.

This routine only supports N <= 3. FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here for larger N.

Returns
N above, or the number of times even elements must be dropped if there is such a number. Otherwise returns zero.

Definition at line 8854 of file X86ISelLowering.cpp.

References llvm::array_lengthof(), llvm::isPowerOf2_32(), isSingleInputShuffleMask(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I8VectorShuffle().

static bool canWidenShuffleElements ( ArrayRef< int Mask,
SmallVectorImpl< int > &  WidenedMask 
)
static

Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.

Appends the mask for wider elements in WidenedMask if valid. Otherwise leaves it in an unspecified state.

NOTE: This must handle normal vector shuffle masks and target vector shuffle masks. The latter have the special property of a '-2' representing a zero-ed lane of a vector.

Definition at line 9235 of file X86ISelLowering.cpp.

References llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by combineX86ShufflesRecursively(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), and lowerVectorShuffle().

static SDValue ChangeVSETULTtoVSETULE ( SDLoc  dl,
SDValue  Op1,
SelectionDAG DAG 
)
static
static bool checkAndUpdateEFLAGSKill ( MachineBasicBlock::iterator  SelectItr,
MachineBasicBlock BB,
const TargetRegisterInfo TRI 
)
static
static bool checkBoolTestAndOrSetCCCombine ( SDValue  Cond,
X86::CondCode CC0,
X86::CondCode CC1,
SDValue Flags,
bool isAnd 
)
static

Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.

Match: (X86or (X86setcc) (X86setcc)) (X86cmp (and (X86setcc) (X86setcc)), 0)

Definition at line 22827 of file X86ISelLowering.cpp.

References AND, llvm::ARMISD::CMP, llvm::dyn_cast(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::ConstantSDNode::isNullValue(), OR, and llvm::ISD::SETCC.

Referenced by PerformCMOVCombine().

static SDValue checkBoolTestSetCCCombine ( SDValue  Cmp,
X86::CondCode CC 
)
static
static bool clobbersFlagRegisters ( const SmallVector< StringRef, 4 > &  AsmPieces)
static
static SDValue CMPEQCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue combineRedundantDWordShuffle ( SDValue  N,
MutableArrayRef< int Mask,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static
static bool combineRedundantHalfShuffle ( SDValue  N,
MutableArrayRef< int Mask,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

Search for a combinable shuffle across a chain ending in pshuflw or pshufhw.

We walk up the chain, skipping shuffles of the other half and looking through shuffles which switch halves trying to find a shuffle of the same pair of dwords.

Definition at line 21417 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPSHUFShuffleMask(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::hasOneUse(), llvm::X86ISD::PSHUFHW, and llvm::X86ISD::PSHUFLW.

Referenced by PerformTargetShuffleCombine().

static SDValue combineShuffleToAddSub ( SDNode N,
SelectionDAG DAG 
)
static

Try to combine a shuffle into a target-specific add-sub node.

We combine this directly on the abstract vector shuffle nodes so it is easier to generically match. We also insert dummy vector shuffle nodes for the operands which explicitly discard the lanes which are unused by this operation to try to flow through the rest of the combiner the fact that they're unused.

Definition at line 21587 of file X86ISelLowering.cpp.

References llvm::X86ISD::ADDSUB, llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), isShuffleEquivalent(), N, llvm::NVPTX::PTXLdStInstCode::V2, and llvm::ISD::VECTOR_SHUFFLE.

Referenced by PerformShuffleCombine().

static bool combineX86ShuffleChain ( SDValue  Op,
SDValue  Root,
ArrayRef< int Mask,
int  Depth,
bool  HasPSHUFB,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Combine an arbitrary chain of shuffles into a single instruction if possible.

This is the leaf of the recursive combinine below. When we have found some chain of single-use x86 shuffle instructions and accumulated the combined shuffle mask represented by them, this will try to pattern match that mask into either a single instruction if there is a special purpose instruction for this operation, or into a PSHUFB instruction which is a fully general instruction but should only be used to replace chains over a certain depth.

Definition at line 20924 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::DL, llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::equals(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::isFloatingPoint(), and llvm::ArrayRef< T >::size().

Referenced by combineX86ShufflesRecursively().

static bool combineX86ShufflesRecursively ( SDValue  Op,
SDValue  Root,
ArrayRef< int RootMask,
int  Depth,
bool  HasPSHUFB,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Fully generic combining of x86 shuffle instructions.

This should be the last combine run over the x86 shuffle instructions. Once they have been fully optimized, this will recursively consider all chains of single-use shuffle instructions, build a generic model of the cumulative shuffle operation, and check for simpler instructions which implement this operation. We use this primarily for two purposes:

1) Collapse generic shuffles to specialized single instructions when equivalent. In most cases, this is just an encoding size win, but sometimes we will collapse multiple generic shuffles into a single special-purpose shuffle. 2) Look for sequences of shuffle instructions with 3 or more total instructions, and replace them with the slightly more expensive SSSE3 PSHUFB instruction if available. We do this as the last combining step to ensure we avoid using PSHUFB if we can implement the shuffle with a suitable short sequence of other instructions. The PHUFB will either use a register or have to read from memory and so is slightly (but only slightly) more expensive than the other shuffle instructions.

Because this is inherently a quadratic operation (for each shuffle in a chain, we recurse up the chain), the depth is limited to 8 instructions. This should never be an issue in practice as the shuffle lowering doesn't produce sequences of more than 8 instructions.

FIXME: We will currently miss some cases where the redundant shuffling would simplify under the threshold for PSHUFB formation because of combine-ordering. To fix this, we should do the redundant instruction combining in this recursive walk.

Definition at line 21122 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, canWidenShuffleElements(), llvm::SmallVectorImpl< T >::clear(), combineX86ShuffleChain(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetShuffleMask(), llvm::MVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), llvm::SDNode::isOnlyUserOf(), isTargetShuffle(), llvm::MVT::isVector(), llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SmallVectorImpl< T >::reserve(), llvm::SmallVectorTemplateCommon< T >::size(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by PerformShuffleCombine().

static SmallBitVector computeZeroableShuffleElements ( ArrayRef< int Mask,
SDValue  V1,
SDValue  V2 
)
static

Compute whether each element of a shuffle is zeroable.

A "zeroable" vector shuffle element is one which can be lowered to zero. Either it is an undef element in the shuffle mask, the element of the input referenced is undef, or the element of the input referenced is known to be zero. Many x86 shuffles can zero lanes cheaply and we often want to handle as many lanes with this technique as possible to simplify the remaining shuffle.

Definition at line 6796 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::ISD::isBuildVectorAllZeros(), llvm::X86::isZeroNode(), llvm::ArrayRef< T >::size(), llvm::ISD::UNDEF, and llvm::NVPTX::PTXLdStInstCode::V2.

Referenced by lowerVectorShuffle(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsInsertPS(), lowerVectorShuffleAsPSHUFB(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), and lowerVectorShuffleWithSSE4A().

static SDValue Concat128BitVectors ( SDValue  V1,
SDValue  V2,
EVT  VT,
unsigned  NumElems,
SelectionDAG DAG,
SDLoc  dl 
)
static

Concat two 128-bit vectors into a 256 bit vector using VINSERTF128 instructions.

This is used because creating CONCAT_VECTOR nodes of BUILD_VECTORS returns a larger BUILD_VECTOR while we're trying to lower large BUILD_VECTORS.

Definition at line 4323 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getUNDEF(), and Insert128BitVector().

Referenced by getOnesVector(), and LowerAVXCONCAT_VECTORS().

static SDValue Concat256BitVectors ( SDValue  V1,
SDValue  V2,
EVT  VT,
unsigned  NumElems,
SelectionDAG DAG,
SDLoc  dl 
)
static

Definition at line 4330 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getUNDEF(), and Insert256BitVector().

Referenced by LowerAVXCONCAT_VECTORS().

static SDValue ConvertI1VectorToInterger ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue CreateCopyOfByValArgument ( SDValue  Src,
SDValue  Dst,
SDValue  Chain,
ISD::ArgFlagsTy  Flags,
SelectionDAG DAG,
SDLoc  dl 
)
static

Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute.

The copy will be passed as a byval function parameter.

Definition at line 2277 of file X86ISelLowering.cpp.

References llvm::ISD::ArgFlagsTy::getByValAlign(), llvm::ISD::ArgFlagsTy::getByValSize(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMemcpy(), and llvm::MVT::i32.

static SDValue EltsFromConsecutiveLoads ( EVT  VT,
ArrayRef< SDValue Elts,
SDLoc DL,
SelectionDAG DAG,
bool  isAfterLegalize 
)
static

Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.

Example: <load i32 *a, load i32 *a+4, undef, undef> -> zextload a

FIXME: we'd also like to handle the case where the last elements are zero rather than undef via VZEXT_LOAD, but we do not detect that case today. There's even a handy isZeroNode for that purpose.

Definition at line 4959 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasAnyUseOfValue(), llvm::MVT::i64, llvm::SelectionDAG::isConsecutiveLoad(), llvm::MemSDNode::isInvariant(), llvm::ISD::isNON_EXTLoad(), llvm::MemSDNode::isNonTemporal(), llvm::TargetLoweringBase::isOperationLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MemSDNode::isVolatile(), llvm::AArch64DB::LD, llvm::ISD::LOAD, llvm::MVT::Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ArrayRef< T >::size(), llvm::ISD::TokenFactor, llvm::ISD::UNDEF, llvm::SelectionDAG::UpdateNodeOperands(), llvm::MVT::v2i64, and llvm::X86ISD::VZEXT_LOAD.

Referenced by LowerINSERT_SUBVECTOR(), and PerformShuffleCombine().

static MachineBasicBlock* EmitMonitor ( MachineInstr MI,
MachineBasicBlock BB,
const X86Subtarget Subtarget 
)
static
static MachineBasicBlock* EmitPCMPSTRI ( MachineInstr MI,
MachineBasicBlock BB,
const TargetInstrInfo TII 
)
static
static MachineBasicBlock* EmitPCMPSTRM ( MachineInstr MI,
MachineBasicBlock BB,
const TargetInstrInfo TII 
)
static
static SDValue EmitTailCallStoreRetAddr ( SelectionDAG DAG,
MachineFunction MF,
SDValue  Chain,
SDValue  RetAddrFrIdx,
EVT  PtrVT,
unsigned  SlotSize,
int  FPDiff,
SDLoc  dl 
)
static

Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0).

Definition at line 2804 of file X86ISelLowering.cpp.

References llvm::MachineFrameInfo::CreateFixedObject(), llvm::MachinePointerInfo::getFixedStack(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), and llvm::SelectionDAG::getStore().

static MachineBasicBlock* EmitXBegin ( MachineInstr MI,
MachineBasicBlock MBB,
const TargetInstrInfo TII 
)
static
static SDValue ExpandHorizontalBinOp ( const SDValue V0,
const SDValue V1,
SDLoc  DL,
SelectionDAG DAG,
unsigned  X86Opcode,
bool  Mode,
bool  isUndefLO,
bool  isUndefHI 
)
static

Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.

This is a helper function of LowerToHorizontalOp(). This function expects two 256-bit vectors called V0 and V1. At first, each vector is split into two separate 128-bit vectors. Then, the resulting 128-bit vectors are used to implement two horizontal binary operations.

The kind of horizontal binary operation is defined by X86Opcode.

Mode specifies how the 128-bit parts of V0 and V1 are passed in input to the two new horizontal binop. When Mode is set, the first horizontal binop dag node would take as input the lower 128-bit of V0 and the upper 128-bit of V0. The second horizontal binop dag node would take as input the lower 128-bit of V1 and the upper 128-bit of V1. Example: HADD V0_LO, V0_HI HADD V1_LO, V1_HI

Otherwise, the first horizontal binop dag node takes as input the lower 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop dag node takes the upper 128-bit of V0 and the upper 128-bit of V1. Example: HADD V0_LO, V1_LO HADD V0_HI, V1_HI

If isUndefLO is set, then the algorithm propagates UNDEF to the lower 128-bits of the result. If isUndefHI is set, then UNDEF is propagated to the upper 128-bits of the result.

Definition at line 5561 of file X86ISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, Extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::AArch64CC::HI, llvm::EVT::is256BitVector(), llvm::AArch64CC::LO, and llvm::ISD::UNDEF.

Referenced by LowerToHorizontalOp().

static SDValue Extract128BitVector ( SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
SDLoc  dl 
)
static

Generate a DAG to grab 128-bits from a vector > 128 bits.

This sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128 or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4 instructions or a simple subregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering EXTRACT_VECTOR_ELT operations easier.

Definition at line 4220 of file X86ISelLowering.cpp.

References ExtractSubVector(), llvm::SDValue::getValueType(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().

Referenced by ExpandHorizontalBinOp(), Lower256IntArith(), Lower256IntVSETCC(), LowerEXTRACT_SUBVECTOR(), LowerShift(), LowerVectorBroadcast(), LowerVectorCTPOP(), PerformSELECTCombine(), PerformShuffleCombine256(), and PerformSTORECombine().

static SDValue Extract256BitVector ( SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
SDLoc  dl 
)
static

Generate a DAG to grab 256-bits from a 512-bit vector.

Definition at line 4228 of file X86ISelLowering.cpp.

References ExtractSubVector(), llvm::SDValue::getValueType(), and llvm::EVT::is512BitVector().

Referenced by LowerEXTRACT_SUBVECTOR().

static SDValue ExtractSubVector ( SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
SDLoc  dl,
unsigned  vectorWidth 
)
static
static bool FuncIsMadeTailCallSafe ( CallingConv::ID  CC,
bool  GuaranteedTailCallOpt 
)
static

Return true if the function is being made into a tailcall target by changing its ABI.

Definition at line 2317 of file X86ISelLowering.cpp.

References IsTailCallConvention().

static ArrayRef<MCPhysReg> get64BitArgumentGPRs ( CallingConv::ID  CallConv,
const X86Subtarget Subtarget 
)
static
static ArrayRef<MCPhysReg> get64BitArgumentXMMs ( MachineFunction MF,
CallingConv::ID  CallConv,
const X86Subtarget Subtarget 
)
static
static unsigned getExtractVEXTRACTImmediate ( SDNode N,
unsigned  vecWidth 
)
static
static SDValue getGatherNode ( unsigned  Opc,
SDValue  Op,
SelectionDAG DAG,
SDValue  Src,
SDValue  Mask,
SDValue  Base,
SDValue  Index,
SDValue  ScaleOp,
SDValue  Chain,
const X86Subtarget Subtarget 
)
static
static unsigned getInsertVINSERTImmediate ( SDNode N,
unsigned  vecWidth 
)
static
static void getMaxByValAlign ( Type Ty,
unsigned MaxAlign 
)
static

Helper for getByValTypeAlignment to determine the desired ByVal argument alignment.

Definition at line 1760 of file X86ISelLowering.cpp.

Referenced by llvm::X86TargetLowering::getByValTypeAlignment().

static SDValue getMOVL ( SelectionDAG DAG,
SDLoc  dl,
EVT  VT,
SDValue  V1,
SDValue  V2 
)
static

getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width.

Definition at line 4365 of file X86ISelLowering.cpp.

References llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().

static SDValue getOnesVector ( MVT  VT,
bool  HasInt256,
SelectionDAG DAG,
SDLoc  dl 
)
static

getOnesVector - Returns a vector of specified type with all bits set.

Always build ones vectors as <4 x i32> or <8 x i32>. For 256-bit types with no AVX2 supprt, use two <4 x i32> inserted in a <8 x i32> appropriately. Then bitcast to their original type, ensuring they get CSE'd.

Definition at line 4341 of file X86ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, Concat128BitVectors(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i32, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::isVector(), llvm_unreachable, llvm::MVT::v4i32, and llvm::MVT::v8i32.

static SDValue getPrefetchNode ( unsigned  Opc,
SDValue  Op,
SelectionDAG DAG,
SDValue  Mask,
SDValue  Base,
SDValue  Index,
SDValue  ScaleOp,
SDValue  Chain 
)
static
static SmallVector<int, 4> getPSHUFShuffleMask ( SDValue  N)
static

Get the PSHUF-style mask from PSHUF node.

This is a very minor wrapper around getTargetShuffleMask to easy forming v4 PSHUF-style masks that can be reused with such instructions.

Definition at line 21241 of file X86ISelLowering.cpp.

References llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetShuffleMask(), llvm_unreachable, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, and llvm::X86ISD::PSHUFLW.

Referenced by combineRedundantDWordShuffle(), combineRedundantHalfShuffle(), and PerformTargetShuffleCombine().

static void getReadPerformanceCounter ( SDNode N,
SDLoc  DL,
SelectionDAG DAG,
const X86Subtarget Subtarget,
SmallVectorImpl< SDValue > &  Results 
)
static
static void getReadTimeStampCounter ( SDNode N,
SDLoc  DL,
unsigned  Opcode,
SelectionDAG DAG,
const X86Subtarget Subtarget,
SmallVectorImpl< SDValue > &  Results 
)
static
static SDValue getScalarMaskingNode ( SDValue  Op,
SDValue  Mask,
SDValue  PreservedSrc,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Creates an SDNode for a predicated scalar operation.

Returns
(X86vselect Mask, Op, PreservedSrc). The mask is comming as MVT::i8 and it should be truncated to MVT::i1 while lowering masking intrinsics. The main difference between ScalarMaskingNode and VectorMaskingNode is using "X86select" instead of "vselect". We just can't create the "vselect" node for a scalar instruction.

Definition at line 15255 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getValueType(), getZeroVector(), llvm::MVT::i1, isAllOnes(), llvm::X86ISD::SELECT, llvm::ISD::TRUNCATE, and llvm::ISD::UNDEF.

Referenced by LowerINTRINSIC_WO_CHAIN().

static SDValue getScalarValueForVectorElement ( SDValue  V,
int  Idx,
SelectionDAG DAG 
)
static
static SDValue getScatterNode ( unsigned  Opc,
SDValue  Op,
SelectionDAG DAG,
SDValue  Src,
SDValue  Mask,
SDValue  Base,
SDValue  Index,
SDValue  ScaleOp,
SDValue  Chain 
)
static
static int getSEHRegistrationNodeSize ( const Function Fn)
static
static SDValue getShuffleScalarElt ( SDNode N,
unsigned  Index,
SelectionDAG DAG,
unsigned  Depth 
)
static
static SDValue getShuffleVectorZeroOrUndef ( SDValue  V2,
unsigned  Idx,
bool  IsZero,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified vector of zero or undef vector.

This produces a shuffle where the low element of V2 is swizzled into the zero/undef vector, landing at element Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).

Definition at line 4403 of file X86ISelLowering.cpp.

References llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), and llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back().

Referenced by llvm::X86TargetLowering::BuildFILD(), and getTargetVShiftNode().

static bool getTargetShuffleMask ( SDNode N,
MVT  VT,
SmallVectorImpl< int > &  Mask,
bool IsUnary 
)
static

getTargetShuffleMask - Calculates the shuffle mask corresponding to the target specific opcode.

Returns true if the Mask could be calculated. Sets IsUnary to true if only uses one source. Note that this will set IsUnary for shuffles which use a single input multiple times, and in those cases it will adjust the mask to only have indices within that single input. FIXME: Add support for Decode*Mask functions that return SM_SentinelZero.

Definition at line 4424 of file X86ISelLowering.cpp.

References llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ISD::BITCAST, llvm::X86ISD::BLENDI, llvm::ISD::BUILD_VECTOR, llvm::DecodeBLENDMask(), llvm::DecodeMOVDDUPMask(), llvm::DecodeMOVHLPSMask(), llvm::DecodeMOVLHPSMask(), llvm::DecodeMOVSHDUPMask(), llvm::DecodeMOVSLDUPMask(), llvm::DecodePALIGNRMask(), llvm::DecodePSHUFBMask(), llvm::DecodePSHUFHWMask(), llvm::DecodePSHUFLWMask(), llvm::DecodePSHUFMask(), llvm::DecodeScalarMoveMask(), llvm::DecodeSHUFPMask(), llvm::DecodeUNPCKHMask(), llvm::DecodeUNPCKLMask(), llvm::DecodeVPERM2X128Mask(), llvm::DecodeVPERMMask(), llvm::dyn_cast(), llvm::SmallVectorBase::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::APInt::getLoBits(), llvm::SDValue::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZExtValue(), llvm::EVT::isInteger(), llvm::EVT::isVector(), llvm_unreachable, llvm::APInt::lshr(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPD, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVLPD, llvm::X86ISD::MOVLPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::X86ISD::SHUFP, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::UNDEF, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMILPI, llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.

Referenced by combineX86ShufflesRecursively(), getPSHUFShuffleMask(), getShuffleScalarElt(), and XFormVExtractWithShuffleIntoLoad().

static SDValue getTargetShuffleNode ( unsigned  Opc,
SDLoc  dl,
EVT  VT,
SDValue  V1,
unsigned  TargetMask,
SelectionDAG DAG 
)
static
static SDValue getTargetShuffleNode ( unsigned  Opc,
SDLoc  dl,
EVT  VT,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG 
)
static
static SDValue getTargetVShiftByConstNode ( unsigned  Opc,
SDLoc  dl,
MVT  VT,
SDValue  SrcOp,
uint64_t  ShiftAmt,
SelectionDAG DAG 
)
static
static SDValue getTargetVShiftNode ( unsigned  Opc,
SDLoc  dl,
MVT  VT,
SDValue  SrcOp,
SDValue  ShAmt,
SelectionDAG DAG 
)
static
static SDValue GetTLSADDR ( SelectionDAG DAG,
SDValue  Chain,
GlobalAddressSDNode GA,
SDValue InFlag,
const EVT  PtrVT,
unsigned  ReturnReg,
unsigned char  OperandFlags,
bool  LocalDynamic = false 
)
static
static int getUnderlyingExtractedFromVec ( SDValue ExtractedFromVec,
SDValue  ExtIdx 
)
static

For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.

Modifies ExtractedFromVec to the real vector and returns the real index.

Definition at line 5230 of file X86ISelLowering.cpp.

References llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), and isUndefOrInRange().

Referenced by buildFromShuffleMostly().

static SDValue getUnpackh ( SelectionDAG DAG,
SDLoc  dl,
MVT  VT,
SDValue  V1,
SDValue  V2 
)
static

getUnpackh - Returns a vector_shuffle node for an unpackh operation.

Definition at line 4388 of file X86ISelLowering.cpp.

References llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().

Referenced by LowerAVXExtend().

static SDValue getUnpackl ( SelectionDAG DAG,
SDLoc  dl,
MVT  VT,
SDValue  V1,
SDValue  V2 
)
static

getUnpackl - Returns a vector_shuffle node for an unpackl operation.

Definition at line 4376 of file X86ISelLowering.cpp.

References llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().

Referenced by llvm::X86TargetLowering::BuildFILD(), and LowerAVXExtend().

static SDValue getV4X86ShuffleImm8ForMask ( ArrayRef< int Mask,
SDLoc  DL,
SelectionDAG DAG 
)
static

Get a 4-lane 8-bit shuffle immediate for a mask.

This helper function produces an 8-bit shuffle immediate corresponding to the ubiquitous shuffle encoding scheme used in x86 instructions for shuffling 4 lanes. It can be used with most of the PSHUF instructions for example.

NB: We rely heavily on "undef" masks preserving the input lane.

Definition at line 6409 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getConstant(), llvm::MVT::i8, and llvm::ArrayRef< T >::size().

Referenced by combineRedundantDWordShuffle(), combineRedundantHalfShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerV8I32VectorShuffle(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), lowerVectorShuffleWithSHUFPS(), and PerformTargetShuffleCombine().

static SDValue getVectorMaskingNode ( SDValue  Op,
SDValue  Mask,
SDValue  PreservedSrc,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue getVShift ( bool  isLeft,
EVT  VT,
SDValue  SrcOp,
unsigned  NumBits,
SelectionDAG DAG,
const TargetLowering TLI,
SDLoc  dl 
)
static
static SDValue getZeroVector ( EVT  VT,
const X86Subtarget Subtarget,
SelectionDAG DAG,
SDLoc  dl 
)
static
static bool hasFPCMov ( unsigned  X86CC)
static

hasFPCMov - is there a floating point cmov for the specific X86 condition code.

Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.

Definition at line 3904 of file X86ISelLowering.cpp.

References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, and llvm::X86::COND_P.

Referenced by PerformCMOVCombine().

static bool hasMFENCE ( const X86Subtarget Subtarget)
static
static bool hasNonFlagsUse ( SDValue  Op)
static
static SDValue Insert128BitVector ( SDValue  Result,
SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
SDLoc  dl 
)
static
static SDValue Insert256BitVector ( SDValue  Result,
SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
SDLoc  dl 
)
static
static SDValue InsertSubVector ( SDValue  Result,
SDValue  Vec,
unsigned  IdxVal,
SelectionDAG DAG,
SDLoc  dl,
unsigned  vectorWidth 
)
static
static bool is128BitLaneCrossingShuffleMask ( MVT  VT,
ArrayRef< int Mask 
)
static

Test whether there are elements crossing 128-bit lanes in this shuffle mask.

X86 divides up its shuffles into in-lane and cross-lane shuffle operations and we routinely test for these.

Definition at line 6321 of file X86ISelLowering.cpp.

References llvm::MVT::getScalarSizeInBits(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV8F32VectorShuffle(), and lowerVectorShuffleByMerging128BitLanes().

static bool is128BitLaneRepeatedShuffleMask ( MVT  VT,
ArrayRef< int Mask,
SmallVectorImpl< int > &  RepeatedMask 
)
static

Test whether a shuffle mask is equivalent within each 128-bit lane.

This checks a shuffle mask to see if it is performing the same 128-bit lane-relative shuffle in each 128-bit lane. This trivially implies that it is also not lane-crossing. It may however involve a blend from the same lane of a second vector.

The specific repeated shuffle mask is populated in RepeatedMask, as it is non-trivial to compute in the face of undef lanes. The representation is not suitable for use with existing 128-bit shuffles as it will contain entries from both V1 and V2 inputs to the wider mask.

Definition at line 6342 of file X86ISelLowering.cpp.

References llvm::MVT::getScalarSizeInBits(), llvm::SmallVectorImpl< T >::resize(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I16VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I32VectorShuffle(), and lowerVectorShuffleAsBlend().

static bool isAllOnes ( SDValue  V)
static
static bool isAndOrOfSetCCs ( SDValue  Op,
unsigned Opc 
)
static
static bool IsCCallConvention ( CallingConv::ID  CC)
static

Return true if the calling convention is a C calling convention.

Definition at line 2296 of file X86ISelLowering.cpp.

References llvm::CallingConv::C, llvm::CallingConv::X86_64_SysV, and llvm::CallingConv::X86_64_Win64.

static bool isHorizontalBinOp ( const BuildVectorSDNode N,
unsigned  Opcode,
SelectionDAG DAG,
unsigned  BaseIdx,
unsigned  LastIdx,
SDValue V0,
SDValue V1 
)
static

Return true if N implements a horizontal binop and return the operands for the horizontal binop into V0 and V1.

This is a helper function of LowerToHorizontalOp(). This function checks that the build_vector N in input implements a horizontal operation. Parameter Opcode defines the kind of horizontal operation to match. For example, if Opcode is equal to ISD::ADD, then this function checks if N implements a horizontal arithmetic add; if instead Opcode is equal to ISD::SUB, then this function checks if this is a horizontal arithmetic sub.

This function only analyzes elements of N whose indices are in range [BaseIdx, LastIdx).

Definition at line 5447 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::EVT::isVector(), and llvm::ISD::UNDEF.

Referenced by LowerToHorizontalOp(), PerformAddCombine(), PerformFADDCombine(), PerformFSUBCombine(), and PerformSubCombine().

static bool isHorizontalBinOp ( SDValue LHS,
SDValue RHS,
bool  IsCommutative 
)
static

Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS.

A horizontal operation performs the binary operation on successive elements of its first operand, then on successive elements of its second operand, returning the resulting values in a vector. For example, if A = < float a0, float a1, float a2, float a3 > and B = < float b0, float b1, float b2, float b3 > then the result of doing a horizontal operation on A and B is A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >. In short, LHS and RHS are inspected to see if LHS op RHS is of the form A horizontal-op B, for some already available A and B, and if so then LHS is set to A, RHS to B, and the routine returns 'true'. Note that the binary operation should have the property that if one of the operands is UNDEF then the result is UNDEF.

Definition at line 24372 of file X86ISelLowering.cpp.

References llvm::ARM_PROC::A, llvm::SmallVectorTemplateCommon< T >::begin(), llvm::ArrayRef< T >::begin(), llvm::CallingConv::C, llvm::ArrayRef< T >::end(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::ISD::UNDEF, and llvm::ISD::VECTOR_SHUFFLE.

static bool isNoopShuffleMask ( ArrayRef< int Mask)
static

Tiny helper function to identify a no-op mask.

This is a somewhat boring predicate function. It checks whether the mask array input, which is assumed to be a single-input shuffle mask of the kind used by the X86 shuffle instructions (not a fully general ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an in-place shuffle are 'no-op's.

Definition at line 6294 of file X86ISelLowering.cpp.

References llvm::ArrayRef< T >::size().

Referenced by lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsUnpack(), and PerformTargetShuffleCombine().

static bool isSequentialOrUndefInRange ( ArrayRef< int Mask,
unsigned  Pos,
unsigned  Size,
int  Low 
)
static

isSequentialOrUndefInRange - Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size].

or is undef.

Definition at line 3997 of file X86ISelLowering.cpp.

References isUndefOrEqual().

Referenced by lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), and lowerVectorShuffleWithSSE4A().

static bool isShuffleEquivalent ( SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
ArrayRef< int ExpectedMask 
)
static

Checks whether a shuffle mask is equivalent to an explicit list of arguments.

This is a fast way to test a shuffle mask against a fixed pattern:

if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }

It returns true if the mask is exactly as wide as the argument list, and each element of the mask is either -1 (signifying undef) or the value given in the argument.

Definition at line 6376 of file X86ISelLowering.cpp.

References llvm::dyn_cast(), llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.

Referenced by combineShuffleToAddSub(), lowerV16F32VectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV2X128VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().

static bool isShuffleFoldableLoad ( SDValue  V)
static

Helper to test for a load that can be folded with x86 shuffles.

This is particularly important because the set of instructions varies significantly based on whether the operand is a load or not.

Definition at line 7325 of file X86ISelLowering.cpp.

References llvm::ISD::BITCAST, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), and llvm::ISD::isNON_EXTLoad().

Referenced by lowerV2F64VectorShuffle(), lowerVectorShuffleAsBroadcast(), and PerformBLENDICombine().

static bool isShuffleHigh128VectorInsertLow ( ShuffleVectorSDNode SVOp)
static

isShuffleHigh128VectorInsertLow - Checks whether the shuffle node is the same as extracting the high 128-bit part of 256-bit vector and then inserting the result into the low part of a new 256-bit vector

Definition at line 20791 of file X86ISelLowering.cpp.

References llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), and isUndefOrEqual().

Referenced by PerformShuffleCombine256().

static bool isShuffleLow128VectorInsertHigh ( ShuffleVectorSDNode SVOp)
static

isShuffleLow128VectorInsertHigh - Checks whether the shuffle node is the same as extracting the low 128-bit part of 256-bit vector and then inserting the result into the high part of a new 256-bit vector

Definition at line 20807 of file X86ISelLowering.cpp.

References llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), and isUndefOrEqual().

Referenced by PerformShuffleCombine256().

static bool isShuffleMaskInputInPlace ( int  Input,
ArrayRef< int Mask 
)
static

Test whether the specified input (0 or 1) is in-place blended by the given mask.

This returns true if the elements from a particular input are already in the slot required by the given mask and require no permutation.

Definition at line 9690 of file X86ISelLowering.cpp.

References llvm::ArrayRef< T >::size().

Referenced by lowerV4F64VectorShuffle(), and lowerV4I64VectorShuffle().

static bool isSingleInputShuffleMask ( ArrayRef< int Mask)
static

Helper function to classify a mask as a single-input mask.

This isn't a generic single-input test because in the vector shuffle lowering we canonicalize single inputs to be the first input operand. This means we can more quickly test for a single input by only checking whether an input from the second operand exists. We also assume that the size of mask corresponds to the size of the input vectors which isn't true in the fully general case.

Definition at line 6309 of file X86ISelLowering.cpp.

Referenced by canLowerByDroppingEvenElements(), lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I32VectorShuffle(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleAsUnpack(), and lowerVectorShuffleByMerging128BitLanes().

static bool isSingleSHUFPSMask ( ArrayRef< int Mask)
static

Test whether this can be lowered with a single SHUFPS instruction.

This is used to disable more specialized lowerings when the shufps lowering will happen to be efficient.

Definition at line 7904 of file X86ISelLowering.cpp.

References llvm::ArrayRef< T >::size().

Referenced by lowerV4F32VectorShuffle().

static bool IsTailCallConvention ( CallingConv::ID  CC)
static

Return true if the calling convention is one that supports tail call optimization.

Definition at line 2290 of file X86ISelLowering.cpp.

References llvm::CallingConv::Fast, llvm::CallingConv::GHC, and llvm::CallingConv::HiPE.

Referenced by FuncIsMadeTailCallSafe().

static bool isTargetShuffle ( unsigned  Opcode)
static
static bool isTruncWithZeroHighBitsInput ( SDValue  V,
SelectionDAG DAG 
)
static
static bool isUndefInRange ( ArrayRef< int Mask,
unsigned  Pos,
unsigned  Size 
)
static

isUndefInRange - Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef.

Definition at line 3975 of file X86ISelLowering.cpp.

Referenced by lowerVectorShuffleAsSpecificZeroOrAnyExtend(), and lowerVectorShuffleWithSSE4A().

static bool isUndefOrEqual ( int  Val,
int  CmpVal 
)
static

isUndefOrEqual - Val is either less than zero (undef) or equal to the specified value.

Definition at line 3990 of file X86ISelLowering.cpp.

Referenced by isSequentialOrUndefInRange(), isShuffleHigh128VectorInsertLow(), isShuffleLow128VectorInsertHigh(), and PerformShuffleCombine256().

static bool isUndefOrInRange ( int  Val,
int  Low,
int  Hi 
)
static

isUndefOrInRange - Return true if Val is undef or if its value falls within the specified range (L, H].

Definition at line 3984 of file X86ISelLowering.cpp.

References llvm::MipsISD::Hi.

Referenced by getUnderlyingExtractedFromVec().

static bool isVEXTRACTIndex ( SDNode N,
unsigned  vecWidth 
)
static

isVEXTRACTIndex - Return true if the specified EXTRACT_SUBVECTOR operand specifies a vector extract that is suitable for instruction that extract 128 or 256 bit vectors

Definition at line 4008 of file X86ISelLowering.cpp.

References llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), and llvm::MVT::getVectorElementType().

Referenced by llvm::X86::isVEXTRACT128Index(), and llvm::X86::isVEXTRACT256Index().

static bool isVINSERTIndex ( SDNode N,
unsigned  vecWidth 
)
static

isVINSERTIndex - Return true if the specified INSERT_SUBVECTOR operand specifies a subvector insert that is suitable for input to insertion of 128 or 256-bit subvectors

Definition at line 4027 of file X86ISelLowering.cpp.

References llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), and llvm::MVT::getVectorElementType().

Referenced by llvm::X86::isVINSERT128Index(), and llvm::X86::isVINSERT256Index().

static bool isX86CCUnsigned ( unsigned  X86CC)
static
static bool isX86LogicalCmp ( SDValue  Op)
static
static bool isXor1OfSetCC ( SDValue  Op)
static
static bool isZero ( SDValue  V)
static

isZero - Returns true if Elt is a constant integer zero

Definition at line 4117 of file X86ISelLowering.cpp.

References llvm::dyn_cast(), and llvm::ConstantSDNode::isNullValue().

Referenced by BUILD_VECTORtoBlendMask(), llvm::X86::isZeroNode(), and lowerVSELECTtoVectorShuffle().

static SDValue lower128BitVectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
MVT  VT,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Dispatching routine to lower various 128-bit x86 vector shuffles.

This routine breaks down the specific type of 128-bit shuffle and dispatches to the lowering routines accordingly.

Definition at line 9204 of file X86ISelLowering.cpp.

References llvm_unreachable, lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4I32VectorShuffle(), lowerV8I16VectorShuffle(), llvm::MVT::SimpleTy, llvm::MVT::v16i8, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4i32, and llvm::MVT::v8i16.

Referenced by lowerVectorShuffle().

static SDValue lower256BitVectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
MVT  VT,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue Lower256IntArith ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue Lower256IntVSETCC ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue lower512BitVectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
MVT  VT,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

High-level routine to lower various 512-bit x86 vector shuffles.

This routine either breaks down the specific type of a 512-bit x86 vector shuffle or splits it into two 256-bit shuffles and fuses the results back together based on the available instructions.

Definition at line 10476 of file X86ISelLowering.cpp.

References llvm::DL, llvm::ShuffleVectorSDNode::getMask(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm_unreachable, lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV32I16VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I64VectorShuffle(), lowerVectorShuffleAsBroadcast(), llvm::MVT::SimpleTy, splitAndLowerVectorShuffle(), llvm::MVT::v16f32, llvm::MVT::v16i32, llvm::MVT::v32i16, llvm::MVT::v64i8, llvm::MVT::v8f64, and llvm::MVT::v8i64.

Referenced by lowerVectorShuffle().

static SDValue LowerADD ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerADDC_ADDE_SUBC_SUBE ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerADJUST_TRAMPOLINE ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerANY_EXTEND ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerAsSplatVectorLoad ( SDValue  SrcOp,
MVT  VT,
SDLoc  dl,
SelectionDAG DAG 
)
static
static SDValue LowerATOMIC_FENCE ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerATOMIC_STORE ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerAVXCONCAT_VECTORS ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerAVXExtend ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue LowerBITCAST ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerBoolVSETCC_AVX512 ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerBuildVectorv16i8 ( SDValue  Op,
unsigned  NonZeros,
unsigned  NumNonZero,
unsigned  NumZero,
SelectionDAG DAG,
const X86Subtarget Subtarget,
const TargetLowering TLI 
)
static
static SDValue LowerBuildVectorv4x32 ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget,
const TargetLowering TLI 
)
static
static SDValue LowerBuildVectorv8i16 ( SDValue  Op,
unsigned  NonZeros,
unsigned  NumNonZero,
unsigned  NumZero,
SelectionDAG DAG,
const X86Subtarget Subtarget,
const TargetLowering TLI 
)
static
static SDValue LowerCMP_SWAP ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerCONCAT_VECTORS ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerCONCAT_VECTORSvXi1 ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerCTLZ ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerCTLZ_ZERO_UNDEF ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerCTPOP ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerCTTZ ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerExtendedLoad ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 14336 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::EXTLOAD, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getExtLoad(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::TargetLoweringBase::getPointerTy(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i8, llvm::ISD::INSERT_VECTOR_ELT, llvm::MVT::integer_valuetypes(), llvm::MVT::isInteger(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::MVT::isVector(), llvm::SPII::Load, llvm::MVT::Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SEXTLOAD, llvm::ISD::SRA, llvm::ISD::TokenFactor, and llvm::X86ISD::VSEXT.

Referenced by llvm::X86TargetLowering::LowerOperation().

static SDValue LowerEXTRACT_SUBVECTOR ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerEXTRACT_VECTOR_ELT_SSE4 ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerFABSorFNEG ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerFCOPYSIGN ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerFGETSIGN ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerFP_EXTEND ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerFSINCOS ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerHorizontalByteSum ( SDValue  V,
MVT  VT,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Compute the horizontal sum of bytes in V for the elements of VT.

Requires V to be a byte vector and VT to be an integer vector type with wider elements than V's type. The width of the elements of VT determines how many bytes of V are summed horizontally to produce each element of the result.

Definition at line 18059 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::BUILD_VECTOR, llvm::DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::i64, llvm::MVT::i8, llvm::X86ISD::PACKUS, llvm::X86ISD::PSADBW, llvm::ISD::SHL, llvm::ISD::SRL, llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by LowerVectorCTPOPBitmath(), and LowerVectorCTPOPInRegLUT().

static SDValue LowerINSERT_SUBVECTOR ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerINTRINSIC_W_CHAIN ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 16068 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::ADX, llvm::X86ISD::CMOV, llvm::COMPRESS_TO_MEM, llvm::X86::COND_B, llvm::X86::COND_NE, llvm::MachineFrameInfo::CreateFixedObject(), llvm::EXPAND_FROM_MEM, llvm::GATHER, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::X86MachineFunctionInfo::getFAIndex(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), getGatherNode(), llvm::MachineFunction::getInfo(), llvm::getIntrinsicWithChain(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::TargetMachine::getMCAsmInfo(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), getPrefetchNode(), getReadPerformanceCounter(), getReadTimeStampCounter(), llvm::X86Subtarget::getRegisterInfo(), llvm::X86TargetLowering::getReturnAddressFrameIndex(), llvm::EVT::getScalarSizeInBits(), getScatterNode(), llvm::X86RegisterInfo::getSlotSize(), llvm::SelectionDAG::getStore(), llvm::MachineFunction::getTarget(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), getVectorMaskingNode(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::Glue, Hint(), llvm::MVT::i64, llvm::MVT::i8, isAllOnes(), llvm_unreachable, LowerSEHRESTOREFRAME(), llvm::ISD::MERGE_VALUES, llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, llvm::MVT::Other, llvm::PREFETCH, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::RDPMC, llvm::RDRAND, llvm::RDSEED, llvm::RDTSC, llvm::MipsISD::Ret, llvm::SCATTER, llvm::X86ISD::SETCC, llvm::X86MachineFunctionInfo::setFAIndex(), llvm::MachineFrameInfo::setFrameAddressIsTaken(), llvm::MachineFrameInfo::setReturnAddressIsTaken(), llvm::IntrinsicData::Type, llvm::MCAsmInfo::usesWindowsCFI(), llvm::TargetLowering::verifyReturnAddressArgumentIsConstant(), llvm::XTEST, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

static SDValue LowerINTRINSIC_WO_CHAIN ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 15326 of file X86ISelLowering.cpp.

References llvm::BLEND, llvm::CMP_MASK, llvm::CMP_MASK_CC, llvm::COMI, llvm::COMPRESS_EXPAND_IN_REG, llvm::X86::COND_A, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_INVALID, llvm::X86::COND_O, llvm::X86::COND_S, llvm::X86::CUR_DIRECTION, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::FMA_OP_MASK, llvm::FMA_OP_MASK3, llvm::FMA_OP_MASKZ, llvm::X86RegisterInfo::getBaseRegister(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MachineModuleInfo::getContext(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::getIntrinsicWithoutChain(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMCSymbol(), llvm::MachineFunction::getMMI(), llvm::Value::getName(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::MCContext::getOrCreateLSDASymbol(), llvm::X86RegisterInfo::getPtrSizedFrameRegister(), llvm::GlobalValue::getRealLinkageName(), llvm::X86Subtarget::getRegisterInfo(), getScalarMaskingNode(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), getVectorMaskingNode(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::X86RegisterInfo::hasBasePointer(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::INTR_TYPE_1OP, llvm::INTR_TYPE_1OP_MASK, llvm::INTR_TYPE_1OP_MASK_RM, llvm::INTR_TYPE_2OP, llvm::INTR_TYPE_2OP_MASK, llvm::INTR_TYPE_2OP_MASK_RM, llvm::INTR_TYPE_3OP, llvm::INTR_TYPE_3OP_MASK, llvm::INTR_TYPE_4OP, llvm::INTR_TYPE_SCALAR_MASK_RM, isAllOnes(), llvm::X86ISD::KORTEST, llvm_unreachable, llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, llvm::X86ISD::PCMPESTRI, llvm::X86ISD::PCMPISTRI, llvm::X86ISD::PTEST, recoverFramePointer(), llvm::report_fatal_error(), llvm::X86ISD::SETCC, llvm::X86ISD::TESTP, TranslateX86CC(), llvm::ISD::TRUNCATE, llvm::IntrinsicData::Type, llvm::MVT::v16i1, llvm::VPERM_3OP_MASK, llvm::VPERM_3OP_MASKZ, llvm::X86ISD::VPERMV, llvm::VSHIFT, llvm::VSHIFT_MASK, llvm::X86ISD::Wrapper, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

static SDValue LowerIntVSETCC_AVX512 ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue LowerLOAD_SUB ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerMGATHER ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerMSCATTER ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerMUL ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerMUL_LOHI ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerREADCYCLECOUNTER ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerSCALAR_TO_VECTOR ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerScalarImmediateShift ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue LowerScalarVariableShift ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue LowerSEHRESTOREFRAME ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerShift ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 17287 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::ANY_EXTEND, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, Extract128BitVector(), llvm::ISD::FP_TO_SINT, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), getTargetShuffleNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::i8, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MVT::isVector(), llvm_unreachable, llvm::MipsISD::Lo, LowerScalarImmediateShift(), LowerScalarVariableShift(), llvm::makeArrayRef(), llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, llvm::ISD::MUL, One, llvm::SDNode::op_begin(), llvm::ISD::OR, llvm::X86ISD::PACKUS, llvm::X86ISD::PCMPGT, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::R13, R2, llvm::ISD::SHL, llvm::APInt::shl(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::SRL, SupportedVectorVarShift(), llvm::ISD::TRUNCATE, llvm::ISD::UNDEF, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::MVT::v8i32, llvm::ISD::VSELECT, llvm::X86ISD::VSHL, llvm::X86ISD::VSRA, llvm::X86ISD::VSRL, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::LowerOperation().

static SDValue LowerShiftParts ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerSIGN_EXTEND ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerSIGN_EXTEND_AVX512 ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerSIGN_EXTEND_VECTOR_INREG ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerSUB ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerToAddSub ( const BuildVectorSDNode BV,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerToHorizontalOp ( const BuildVectorSDNode BV,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerToTLSExecModel ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT,
TLSModel::Model  model,
bool  is64Bit,
bool  isPIC 
)
static
static SDValue LowerToTLSGeneralDynamicModel32 ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT 
)
static
static SDValue LowerToTLSGeneralDynamicModel64 ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT 
)
static
static SDValue LowerToTLSLocalDynamicModel ( GlobalAddressSDNode GA,
SelectionDAG DAG,
const EVT  PtrVT,
bool  is64Bit 
)
static
static SDValue lowerUINT_TO_FP_vXi32 ( SDValue  Op,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue lowerV16F32VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV16I16VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV16I32VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV16I8VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Generic lowering of v16i8 shuffles.

This is a hybrid strategy to lower v16i8 vectors. It first attempts to detect any complexity reducing interleaving. If that doesn't help, it uses UNPCK to spread the i8 elements across two i16-element vectors, and uses the existing lowering for v8i16 blends on each half, finally PACK-ing them back together.

Definition at line 8908 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::sys::path::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), canLowerByDroppingEvenElements(), llvm::DL, llvm::sys::path::end(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), I, isShuffleEquivalent(), isSingleInputShuffleMask(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPSHUFB(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsUnpack(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), llvm::X86ISD::PACKUS, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, and llvm::MVT::v8i16.

Referenced by lower128BitVectorShuffle().

static SDValue lowerV2F64VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Handle lowering of 2-lane 64-bit floating point shuffles.

This is the basis function for the 2-lane 64-bit shuffles as we have full support for floating point shuffles but not integer shuffles. These instructions will incur a domain crossing penalty on some chips though so it is better to avoid lowering through this for integer vectors where possible.

Definition at line 7711 of file X86ISelLowering.cpp.

References llvm::DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getScalarValueForVectorElement(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i8, isShuffleEquivalent(), isShuffleFoldableLoad(), isSingleInputShuffleMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsElementInsertion(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVLPD, llvm::X86ISD::MOVSD, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v2f64, and llvm::X86ISD::VPERMILPI.

Referenced by lower128BitVectorShuffle().

static SDValue lowerV2I64VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV2X128VectorShuffle ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV32I16VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Handle lowering of 32-lane 16-bit integer shuffles.

Definition at line 10440 of file X86ISelLowering.cpp.

References llvm::DL, llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasBWI(), splitAndLowerVectorShuffle(), and llvm::MVT::v32i16.

Referenced by lower512BitVectorShuffle().

static SDValue lowerV32I8VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV4F32VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV4F64VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV4I32VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV4I64VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV64I8VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Handle lowering of 64-lane 8-bit integer shuffles.

Definition at line 10456 of file X86ISelLowering.cpp.

References llvm::DL, llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasBWI(), splitAndLowerVectorShuffle(), and llvm::MVT::v64i8.

Referenced by lower512BitVectorShuffle().

static SDValue lowerV8F32VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV8F64VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV8I16GeneralSingleInputVectorShuffle ( SDLoc  DL,
MVT  VT,
SDValue  V,
MutableArrayRef< int Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.

The lowering strategy is to try to form pairs of input lanes which are targeted at the same half of the final vector, and then use a dword shuffle to place them onto the right half, and finally unpack the paired lanes into their final position.

The exact breakdown of how to form these dword pairs and align them on the correct sides is really tricky. See the comments within the function for more of the details.

This code also handles repeated 128-bit lanes of v8i16 shuffles, but each lane must shuffle the exact same way. In fact, you must pass a v8 Mask to this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16 vector, form the analogous 128-bit 8-element Mask.

Definition at line 8218 of file X86ISelLowering.cpp.

References llvm::sys::path::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), llvm::MutableArrayRef< T >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::data(), llvm::ArrayRef< T >::empty(), llvm::sys::path::end(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::MutableArrayRef< T >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i16, llvm::MVT::i32, isNoopShuffleMask(), llvm_unreachable, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::MutableArrayRef< T >::slice(), std::swap(), and llvm::MVT::v8i16.

Referenced by lowerV16I16VectorShuffle(), and lowerV8I16VectorShuffle().

static SDValue lowerV8I16VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Generic lowering of 8-lane i16 shuffles.

This handles both single-input shuffles and combined shuffle/blends with two inputs. The single input shuffles are immediately delegated to a dedicated lowering routine.

The blends are lowered in one of three fundamental ways. If there are few enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle of the input is significantly cheaper when lowered as an interleaving of the two inputs, try to interleave them. Otherwise, blend the low and high halves of the inputs separately (making them have relatively few inputs) and then concatenate them.

Definition at line 8715 of file X86ISelLowering.cpp.

References llvm::MutableArrayRef< T >::begin(), llvm::DL, llvm::MutableArrayRef< T >::end(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), isShuffleEquivalent(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPSHUFB(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsUnpack(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::MVT::v8i16.

Referenced by lower128BitVectorShuffle().

static SDValue lowerV8I32VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerV8I64VectorShuffle ( SDValue  Op,
SDValue  V1,
SDValue  V2,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVACOPY ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVectorAllZeroTest ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVectorBroadcast ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

LowerVectorBroadcast - Attempt to use the vbroadcast instruction to generate a splat value for the following cases:

  1. A splat BUILD_VECTOR which uses a single scalar load, or a constant.
  2. A splat shuffle which uses a scalar_to_vector node which comes from a scalar load, or a constant. The VBROADCAST node is returned when a pattern is found, or SDValue() otherwise.

Definition at line 5070 of file X86ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::ISD::Constant, llvm::ISD::ConstantFP, llvm::HexagonISD::CP, Extract128BitVector(), F(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasInt256(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasVLX(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::ShuffleVectorSDNode::isSplat(), llvm::EVT::isVector(), llvm::Attribute::OptimizeForSize, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::VBROADCAST, and llvm::ISD::VECTOR_SHUFFLE.

static SDValue LowerVectorCTPOP ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVectorCTPOPBitmath ( SDValue  Op,
SDLoc  DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVectorCTPOPInRegLUT ( SDValue  Op,
SDLoc  DL,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffle ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsBitBlend ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG 
)
static

Try to emit a blend instruction for a shuffle using bit math.

This is used as a fallback approach when first class blend instructions are unavailable. Currently it is only suitable for integer vectors, but could be generalized for floating point vectors if desirable.

Definition at line 6430 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::ISD::BUILD_VECTOR, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::MVT::i64, llvm::MVT::isInteger(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I8VectorShuffle(), and lowerV8I16VectorShuffle().

static SDValue lowerVectorShuffleAsBitMask ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsBlend ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to emit a blend instruction for a shuffle.

This doesn't do any checks for the availability of instructions for blending these values. It relies on the availability of the X86ISD::BLENDI pattern to be matched in the backend with the type given. What it does check for is that the shuffle mask is in fact a blend.

Definition at line 6462 of file X86ISelLowering.cpp.

References llvm::X86ISD::BLENDI, llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i8, is128BitLaneRepeatedShuffleMask(), llvm_unreachable, llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::MVT::SimpleTy, llvm::ArrayRef< T >::size(), llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8f32, llvm::MVT::v8i16, llvm::MVT::v8i32, and llvm::ISD::VSELECT.

Referenced by lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV2X128VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().

static SDValue lowerVectorShuffleAsBlendAndPermute ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG 
)
static

Try to lower as a blend of elements from two inputs followed by a single-input permutation.

This matches the pattern where we can blend elements from two inputs and then reduce the shuffle to a single-input permutation.

Definition at line 6591 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), and llvm::ArrayRef< T >::size().

Referenced by lowerV4F32VectorShuffle(), and lowerVectorShuffleAsDecomposedShuffleBlend().

static SDValue lowerVectorShuffleAsBroadcast ( SDLoc  DL,
MVT  VT,
SDValue  V,
ArrayRef< int Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsByteRotate ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to lower a vector shuffle as a byte rotation.

SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary byte-rotation of the concatenation of two vectors; pre-SSSE3 can use a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will try to generically lower a vector shuffle through such an pattern. It does not check for the profitability of lowering either as PALIGNR or PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form. This matches shuffle vectors that look like:

v8i16 [11, 12, 13, 14, 15, 0, 1, 2]

Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.

Definition at line 6675 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSSE3(), llvm::MipsISD::Hi, llvm::MVT::i8, isNoopShuffleMask(), llvm::MipsISD::Lo, llvm::ISD::OR, llvm::X86ISD::PALIGNR, llvm::ArrayRef< T >::size(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.

Referenced by lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().

static SDValue lowerVectorShuffleAsDecomposedShuffleBlend ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG 
)
static

Generic routine to decompose a shuffle and blend into indepndent blends and permutes.

This matches the extremely common pattern for handling combined shuffle+blend operations on newer X86 ISAs where we have very fast blend operations. It will try to pick the best arrangement of shuffles and blends.

Definition at line 6625 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), isNoopShuffleMask(), lowerVectorShuffleAsBlendAndPermute(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), lowerVectorShuffleAsLanePermuteAndBlend(), and lowerVectorShuffleAsSplitOrBlend().

static SDValue lowerVectorShuffleAsElementInsertion ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsInsertPS ( SDValue  Op,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsLanePermuteAndBlend ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG 
)
static

Lower a vector shuffle crossing multiple 128-bit lanes as a permutation and blend of those lanes.

This essentially blends the out-of-lane inputs to each lane into the lane from a permuted copy of the vector. This lowering strategy results in four instructions in the worst case for a single-input cross lane shuffle which is lower than any other fully general cross-lane shuffle strategy I'm aware of. Special cases for each particular shuffle pattern should be handled prior to trying this lowering.

Definition at line 9476 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::i8, isSingleInputShuffleMask(), lowerVectorShuffleAsDecomposedShuffleBlend(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ArrayRef< T >::size(), splitAndLowerVectorShuffle(), and llvm::X86ISD::VPERM2X128.

Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), and lowerV8F32VectorShuffle().

static SDValue lowerVectorShuffleAsPSHUFB ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG,
bool V1InUse,
bool V2InUse 
)
static
static SDValue lowerVectorShuffleAsShift ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG 
)
static

Try to lower a vector shuffle as a bit shift (shifts in zeros).

Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function matches elements from one of the input vectors shuffled to the left or right with zeroable elements 'shifted in'. It handles both the strictly bit-wise element shifts and the byte shift across an entire 128-bit double quad word lane.

PSHL : (little-endian) left bit shift. [ zz, 0, zz, 2 ] [ -1, 4, zz, -1 ] PSRL : (little-endian) right bit shift. [ 1, zz, 3, zz] [ -1, -1, 7, zz] PSLLDQ : (little-endian) left byte shift [ zz, 0, 1, 2, 3, 4, 5, 6] [ zz, zz, -1, -1, 2, 3, 4, -1] [ zz, zz, zz, zz, zz, zz, -1, 1] PSRLDQ : (little-endian) right byte shift [ 5, 6, 7, zz, zz, zz, zz, zz] [ -1, 5, 6, 7, zz, zz, zz, zz] [ 1, 2, -1, -1, -1, -1, zz, zz]

Definition at line 6897 of file X86ISelLowering.cpp.

References computeZeroableShuffleElements(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i8, isSequentialOrUndefInRange(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ArrayRef< T >::size(), llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VSRLI.

Referenced by lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().

static SDValue lowerVectorShuffleAsSpecificZeroOrAnyExtend ( SDLoc  DL,
MVT  VT,
int  Scale,
bool  AnyExt,
SDValue  InputV,
ArrayRef< int Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleAsSplitOrBlend ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG 
)
static

Either split a vector in halves or decompose the shuffles and the blend.

This is provided as a good fallback for many lowerings of non-single-input shuffles with more than one 128-bit lane. In those cases, we want to select between splitting the shuffle into 128-bit components and stitching those back together vs. extracting the single-input shuffles and blending those results.

Definition at line 9417 of file X86ISelLowering.cpp.

References llvm::SmallBitVector::count(), llvm::MVT::getSizeInBits(), isSingleInputShuffleMask(), lowerVectorShuffleAsDecomposedShuffleBlend(), llvm::SmallBitVector::resize(), llvm::ArrayRef< T >::size(), and splitAndLowerVectorShuffle().

Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), and lowerV8F32VectorShuffle().

static SDValue lowerVectorShuffleAsUnpack ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG 
)
static

Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.

This specifically targets cases where we end up with alternating between the two inputs, and so can permute them into something that feeds a single UNPCK instruction. Note that this routine only targets integer vectors because for floating point vectors we have a generalized SHUFPS lowering strategy that handles everything that doesn't exactly match an unpack, making this clever lowering unnecessary.

Definition at line 7598 of file X86ISelLowering.cpp.

References llvm::SmallVectorImpl< T >::assign(), llvm::ArrayRef< T >::begin(), llvm::ArrayRef< T >::end(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), isSingleInputShuffleMask(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.

Referenced by lowerV16I8VectorShuffle(), lowerV4I32VectorShuffle(), and lowerV8I16VectorShuffle().

static SDValue lowerVectorShuffleAsZeroOrAnyExtend ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Try to lower a vector shuffle as a zero extension on any microarch.

This routine will try to do everything in its power to cleverly lower a shuffle which happens to match the pattern of a zero extend. It doesn't check for the profitability of this lowering, it tries to aggressively match this pattern. It will use all of the micro-architectural details it can to emit an efficient lowering. It handles both blends with all-zero inputs to explicitly zero-extend and undef-lanes (sometimes undef due to masking out later).

The reason we have dedicated lowering for zext-style shuffles is that they are both incredibly common and often quite performance sensitive.

Definition at line 7202 of file X86ISelLowering.cpp.

References llvm::tgtok::Bits, computeZeroableShuffleElements(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), isSequentialOrUndefInRange(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), llvm::ArrayRef< T >::size(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, and llvm::X86ISD::VZEXT_MOVL.

Referenced by lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().

static SDValue lowerVectorShuffleByMerging128BitLanes ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.

This will only succeed when the result of fixing the 128-bit lanes results in a single-input non-lane-crossing shuffle with a repeating shuffle mask in each 128-bit lanes. This handles many cases where we can quickly blend away the lane crosses early and then use simpler shuffles within each lane.

FIXME: It might be worthwhile at some point to support this without requiring the 128-bit lane-relative shuffles to be repeating, but currently in x86 only floating point has interesting non-repeating shuffles, and even those are still marginally more expensive.

Definition at line 9614 of file X86ISelLowering.cpp.

References llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MVT::i64, is128BitLaneCrossingShuffleMask(), llvm::MVT::isFloatingPoint(), isSingleInputShuffleMask(), llvm::SmallVectorImpl< T >::resize(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().

static SDValue lowerVectorShuffleWithSHUFPD ( SDLoc  DL,
MVT  VT,
ArrayRef< int Mask,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG 
)
static
static SDValue lowerVectorShuffleWithSHUFPS ( SDLoc  DL,
MVT  VT,
ArrayRef< int Mask,
SDValue  V1,
SDValue  V2,
SelectionDAG DAG 
)
static

Lower a vector shuffle using the SHUFPS instruction.

This is a helper routine dedicated to lowering vector shuffles using SHUFPS. It makes no assumptions about whether this is the best lowering, it simply uses it.

Definition at line 7923 of file X86ISelLowering.cpp.

References llvm::ArrayRef< T >::begin(), llvm::ArrayRef< T >::end(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), llvm::X86ISD::SHUFP, std::swap(), and llvm::NVPTX::PTXLdStInstCode::V2.

Referenced by lowerV4F32VectorShuffle(), and lowerV8F32VectorShuffle().

static SDValue lowerVectorShuffleWithSSE4A ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG 
)
static
static SDValue lowerVSELECTtoVectorShuffle ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerVSETCC ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

Definition at line 13532 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, ChangeVSETULTtoVSETULE(), llvm::X86ISD::CMPM, llvm::X86ISD::CMPP, EQ, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOperand(), llvm::APInt::getSignBit(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::AArch64CC::GT, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE42(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::MVT::isFloatingPoint(), llvm_unreachable, Lower256IntVSETCC(), LowerBoolVSETCC_AVX512(), LowerIntVSETCC_AVX512(), llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::X86ISD::SUBUS, std::swap(), translateX86FSETCC(), llvm::ISD::TRUNCATE, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::MVT::v2i64, llvm::MVT::v4i32, and llvm::ISD::XOR.

static SDValue LowerXALUO ( SDValue  Op,
SelectionDAG DAG 
)
static
static SDValue LowerZERO_EXTEND ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static SDValue LowerZERO_EXTEND_AVX512 ( SDValue  Op,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static
static bool matchAsm ( StringRef  S,
ArrayRef< const char * >  Pieces 
)
static
static bool MatchingStackOffset ( SDValue  Arg,
unsigned  Offset,
ISD::ArgFlagsTy  Flags,
MachineFrameInfo MFI,
const MachineRegisterInfo MRI,
const X86InstrInfo TII 
)
static
static std::pair<unsigned, bool> matchIntegerMINMAX ( SDValue  Cond,
EVT  VT,
SDValue  LHS,
SDValue  RHS,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue MaterializeSETB ( SDLoc  DL,
SDValue  EFLAGS,
SelectionDAG DAG,
MVT  VT 
)
static
static bool MayFoldIntoStore ( SDValue  Op)
static
static bool MayFoldLoad ( SDValue  Op)
static
static SDValue NarrowVectorLoadToElement ( LoadSDNode Load,
unsigned  Index,
SelectionDAG DAG 
)
static
static SDValue OptimizeConditionalInDecrement ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue PerformADCCombine ( SDNode N,
SelectionDAG DAG,
X86TargetLowering::DAGCombinerInfo &  DCI 
)
static
static SDValue PerformAddCombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue PerformAndCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformBITCASTCombine ( SDNode N,
SelectionDAG DAG 
)
static

Detect bitcasts between i32 to x86mmx low word.

Since MMX types are special and don't usually play with other vector types, it's better to handle them early to be sure we emit efficient code by avoiding store-load conversions.

Definition at line 21844 of file X86ISelLowering.cpp.

References llvm::ISD::BUILD_VECTOR, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::ConstantSDNode::getZExtValue(), and llvm::X86ISD::MMX_MOVW2D.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue PerformBLENDICombine ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue PerformBrCondCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformBTCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static
static SDValue PerformCMOVCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformEXTRACT_VECTOR_ELTCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

PerformEXTRACT_VECTOR_ELTCombine - Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.

For i686, the best sequence is apparently storing the value and loading scalars back, while for x64 we should use 64-bit extracts and shifts.

Definition at line 21864 of file X86ISelLowering.cpp.

References ADD, llvm::SmallVectorTemplateCommon< T >::begin(), llvm::ISD::BITCAST, llvm::SelectionDAG::CreateStackTemporary(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SDValue::getResNo(), llvm::TargetLoweringBase::getShiftAmountTy(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::TargetLoweringBase::getVectorIdxTy(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::TargetLoweringBase::isOperationLegal(), llvm::X86ISD::MMX_MOVD2W, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), XFormVExtractWithShuffleIntoLoad(), and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue PerformFADDCombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue PerformFANDCombine ( SDNode N,
SelectionDAG DAG 
)
static

Do target-specific dag combines on X86ISD::FAND nodes.

Definition at line 24551 of file X86ISelLowering.cpp.

References llvm::SDNode::getOperand().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue PerformFANDNCombine ( SDNode N,
SelectionDAG DAG 
)
static

Do target-specific dag combines on X86ISD::FANDN nodes.

Definition at line 24566 of file X86ISelLowering.cpp.

References llvm::SDNode::getOperand().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue PerformFMACombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue PerformFMinFMaxCombine ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue PerformFORCombine ( SDNode N,
SelectionDAG DAG 
)
static

Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.

Definition at line 24514 of file X86ISelLowering.cpp.

References llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SDNode::getOpcode(), and llvm::SDNode::getOperand().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue PerformFSUBCombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue PerformINSERTPSCombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue performIntegerAbsCombine ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue PerformINTRINSIC_WO_CHAINCombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue PerformISDSETCCCombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue PerformLOADCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformMLOADCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformMSTORECombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue PerformMulCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static
static SDValue PerformOrCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformSELECTCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

PerformSELECTCombine - Do target-specific dag combines on SELECT and VSELECT nodes.

Definition at line 22146 of file X86ISelLowering.cpp.

References ADD, llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), AND, llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), llvm::ISD::CONCAT_VECTORS, Extract128BitVector(), llvm::AArch64ISD::FMAX, llvm::AArch64ISD::FMIN, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasVLX(), I, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SelectionDAG::isEqualTo(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::SelectionDAG::isKnownNeverZero(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), matchIntegerMINMAX(), llvm::ISD::MUL, llvm::TargetMachine::Options, OR, Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::MipsISD::Ret, llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::X86ISD::SHRUNKBLEND, llvm::ISD::SIGN_EXTEND, llvm::TargetLowering::SimplifyDemandedBits(), SUB, llvm::X86ISD::SUBUS, std::swap(), transformVSELECTtoBlendVECTOR_SHUFFLE(), llvm::TargetOptions::UnsafeFPMath, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::ISD::VSELECT, XOR, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue PerformSETCCCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformSExtCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformShiftCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

PerformShiftCombine - Combine shifts.

Definition at line 23309 of file X86ISelLowering.cpp.

References llvm::SDNode::getOpcode(), performShiftToAllZeros(), PerformSHLCombine(), llvm::ISD::SHL, and llvm::ISD::SRA.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue performShiftToAllZeros ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Returns a vector of 0s if the node in input is a vector logical shift by a constant amount which is known to be bigger than or equal to the vector element size in bits.

Definition at line 23281 of file X86ISelLowering.cpp.

References llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), getZeroVector(), llvm::X86Subtarget::hasInt256(), llvm::APInt::trunc(), and llvm::APInt::uge().

Referenced by PerformShiftCombine().

static SDValue PerformSHLCombine ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue PerformShuffleCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

PerformShuffleCombine - Performs several different shuffle combines.

Definition at line 21636 of file X86ISelLowering.cpp.

References ADD, llvm::ISD::BITCAST, combineShuffleToAddSub(), combineX86ShufflesRecursively(), EltsFromConsecutiveLoads(), llvm::ISD::FADD, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::ShuffleVectorSDNode::getMask(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), getShuffleScalarElt(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasFp256(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE3(), llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isOperationLegal(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::AArch64DB::LD, llvm::ISD::MUL, N, PerformShuffleCombine256(), PerformTargetShuffleCombine(), llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), SUB, llvm::ISD::UNDEF, and llvm::ISD::VECTOR_SHUFFLE.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue PerformShuffleCombine256 ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformSIGN_EXTEND_INREGCombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue PerformSINT_TO_FPCombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue PerformSTORECombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

PerformSTORECombine - Do target-specific dag combines on STORE nodes.

Definition at line 24111 of file X86ISelLowering.cpp.

References ADD, Extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), llvm::MachinePointerInfo::getWithOffset(), llvm::Function::hasFnAttribute(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::is64Bit(), llvm::MemSDNode::isInvariant(), llvm::MemSDNode::isNonTemporal(), llvm::ISD::isNormalLoad(), llvm::isPowerOf2_32(), llvm::StoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86Subtarget::isUnalignedMem32Slow(), isVolatile(), llvm::MemSDNode::isVolatile(), fuzzer::min(), llvm::MinAlign(), N, Other, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ISD::TokenFactor, and llvm::X86Subtarget::useSoftFloat().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

static SDValue PerformSubCombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue PerformTargetShuffleCombine ( SDValue  N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformUINT_TO_FPCombine ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static SDValue performVectorCompareAndMaskUnaryOpCombine ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue PerformVZEXT_MOVLCombine ( SDNode N,
SelectionDAG DAG 
)
static
static SDValue performVZEXTCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformXorCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue PerformZExtCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue recoverFramePointer ( SelectionDAG DAG,
const Function Fn,
SDValue  EntryEBP 
)
static

When the 32-bit MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP.

Here's the math: RegNodeBase = EntryEBP - RegNodeSize ParentFP = RegNodeBase - RegNodeFrameOffset Subtracting RegNodeSize takes us to the offset of the registration node, and subtracting the offset (negative on x86) takes us back to the parent FP.

Definition at line 15294 of file X86ISelLowering.cpp.

References llvm::SelectionDAG::getConstant(), llvm::MachineModuleInfo::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMCSymbol(), llvm::MachineFunction::getMMI(), llvm::Value::getName(), llvm::SelectionDAG::getNode(), llvm::MCContext::getOrCreateParentFrameOffsetSymbol(), llvm::TargetLoweringBase::getPointerTy(), llvm::GlobalValue::getRealLinkageName(), getSEHRegistrationNodeSize(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasPersonalityFn(), llvm::ISD::LOCAL_RECOVER, and llvm::ISD::SUB.

Referenced by LowerINTRINSIC_WO_CHAIN(), and LowerSEHRESTOREFRAME().

static SDValue splitAndLowerVectorShuffle ( SDLoc  DL,
MVT  VT,
SDValue  V1,
SDValue  V2,
ArrayRef< int Mask,
SelectionDAG DAG 
)
static
STATISTIC ( NumTailCalls  ,
"Number of tail calls"   
)
static bool SupportedVectorShiftWithBaseAmnt ( MVT  VT,
const X86Subtarget Subtarget,
unsigned  Opcode 
)
static

Definition at line 17030 of file X86ISelLowering.cpp.

References SupportedVectorShiftWithImm().

Referenced by LowerScalarVariableShift().

static bool SupportedVectorShiftWithImm ( MVT  VT,
const X86Subtarget Subtarget,
unsigned  Opcode 
)
static
static bool SupportedVectorVarShift ( MVT  VT,
const X86Subtarget Subtarget,
unsigned  Opcode 
)
static
static SDValue transformVSELECTtoBlendVECTOR_SHUFFLE ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static
static unsigned TranslateX86CC ( ISD::CondCode  SetCCOpcode,
SDLoc  DL,
bool  isFP,
SDValue LHS,
SDValue RHS,
SelectionDAG DAG 
)
static
static int translateX86FSETCC ( ISD::CondCode  SetCCOpcode,
SDValue Op0,
SDValue Op1 
)
static
static SDValue VectorZextCombine ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue WidenMaskArithmetic ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static
static SDValue XFormVExtractWithShuffleIntoLoad ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

Variable Documentation

cl::opt<bool> ExperimentalVectorWideningLegalization("x86-experimental-vector-widening-legalization", cl::init(false), cl::desc("Enable an experimental vector type legalization through widening ""rather than promotion."), cl::Hidden)
static