|
LLVM
3.7.0
|
#include "X86ISelLowering.h"#include "Utils/X86ShuffleDecode.h"#include "X86CallingConv.h"#include "X86FrameLowering.h"#include "X86InstrBuilder.h"#include "X86MachineFunctionInfo.h"#include "X86TargetMachine.h"#include "X86TargetObjectFile.h"#include "llvm/ADT/SmallBitVector.h"#include "llvm/ADT/SmallSet.h"#include "llvm/ADT/Statistic.h"#include "llvm/ADT/StringExtras.h"#include "llvm/ADT/StringSwitch.h"#include "llvm/CodeGen/IntrinsicLowering.h"#include "llvm/CodeGen/MachineFrameInfo.h"#include "llvm/CodeGen/MachineFunction.h"#include "llvm/CodeGen/MachineInstrBuilder.h"#include "llvm/CodeGen/MachineJumpTableInfo.h"#include "llvm/CodeGen/MachineModuleInfo.h"#include "llvm/CodeGen/MachineRegisterInfo.h"#include "llvm/CodeGen/WinEHFuncInfo.h"#include "llvm/IR/CallSite.h"#include "llvm/IR/CallingConv.h"#include "llvm/IR/Constants.h"#include "llvm/IR/DerivedTypes.h"#include "llvm/IR/Function.h"#include "llvm/IR/GlobalAlias.h"#include "llvm/IR/GlobalVariable.h"#include "llvm/IR/Instructions.h"#include "llvm/IR/Intrinsics.h"#include "llvm/MC/MCAsmInfo.h"#include "llvm/MC/MCContext.h"#include "llvm/MC/MCExpr.h"#include "llvm/MC/MCSymbol.h"#include "llvm/Support/CommandLine.h"#include "llvm/Support/Debug.h"#include "llvm/Support/ErrorHandling.h"#include "llvm/Support/MathExtras.h"#include "llvm/Target/TargetOptions.h"#include "X86IntrinsicsInfo.h"#include <bitset>#include <numeric>#include <cctype>#include "X86GenCallingConv.inc"Go to the source code of this file.
Macros | |
| #define | DEBUG_TYPE "x86-isel" |
Enumerations | |
| enum | StructReturnType { NotStructReturn, RegStructReturn, StackStructReturn } |
| CallIsStructReturn - Determines whether a call uses struct return semantics. More... | |
Functions | |
| STATISTIC (NumTailCalls,"Number of tail calls") | |
| static SDValue | getMOVL (SelectionDAG &DAG, SDLoc dl, EVT VT, SDValue V1, SDValue V2) |
| getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width. More... | |
| static void | getMaxByValAlign (Type *Ty, unsigned &MaxAlign) |
| Helper for getByValTypeAlignment to determine the desired ByVal argument alignment. More... | |
| static StructReturnType | callIsStructReturn (const SmallVectorImpl< ISD::OutputArg > &Outs) |
| static StructReturnType | argsAreStructReturn (const SmallVectorImpl< ISD::InputArg > &Ins) |
| Determines whether a function uses struct return semantics. More... | |
| static SDValue | CreateCopyOfByValArgument (SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, SDLoc dl) |
| Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute. More... | |
| static bool | IsTailCallConvention (CallingConv::ID CC) |
| Return true if the calling convention is one that supports tail call optimization. More... | |
| static bool | IsCCallConvention (CallingConv::ID CC) |
| Return true if the calling convention is a C calling convention. More... | |
| static bool | FuncIsMadeTailCallSafe (CallingConv::ID CC, bool GuaranteedTailCallOpt) |
| Return true if the function is being made into a tailcall target by changing its ABI. More... | |
| static ArrayRef< MCPhysReg > | get64BitArgumentGPRs (CallingConv::ID CallConv, const X86Subtarget *Subtarget) |
| static ArrayRef< MCPhysReg > | get64BitArgumentXMMs (MachineFunction &MF, CallingConv::ID CallConv, const X86Subtarget *Subtarget) |
| static SDValue | EmitTailCallStoreRetAddr (SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue RetAddrFrIdx, EVT PtrVT, unsigned SlotSize, int FPDiff, SDLoc dl) |
| Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0). More... | |
| static bool | MatchingStackOffset (SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo *MFI, const MachineRegisterInfo *MRI, const X86InstrInfo *TII) |
| MatchingStackOffset - Return true if the given stack call argument is already available in the same position (relatively) of the caller's incoming argument stack. More... | |
| static bool | MayFoldLoad (SDValue Op) |
| static bool | MayFoldIntoStore (SDValue Op) |
| static bool | isTargetShuffle (unsigned Opcode) |
| static SDValue | getTargetShuffleNode (unsigned Opc, SDLoc dl, EVT VT, SDValue V1, unsigned TargetMask, SelectionDAG &DAG) |
| static SDValue | getTargetShuffleNode (unsigned Opc, SDLoc dl, EVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| static bool | isX86CCUnsigned (unsigned X86CC) |
| Return true if the condition is an unsigned comparison operation. More... | |
| static unsigned | TranslateX86CC (ISD::CondCode SetCCOpcode, SDLoc DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) |
| TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86 specific condition code, returning the condition code and the LHS/RHS of the comparison to make. More... | |
| static bool | hasFPCMov (unsigned X86CC) |
| hasFPCMov - is there a floating point cmov for the specific X86 condition code. More... | |
| static bool | isUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size) |
| isUndefInRange - Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef. More... | |
| static bool | isUndefOrInRange (int Val, int Low, int Hi) |
| isUndefOrInRange - Return true if Val is undef or if its value falls within the specified range (L, H]. More... | |
| static bool | isUndefOrEqual (int Val, int CmpVal) |
| isUndefOrEqual - Val is either less than zero (undef) or equal to the specified value. More... | |
| static bool | isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low) |
| isSequentialOrUndefInRange - Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size]. More... | |
| static bool | isVEXTRACTIndex (SDNode *N, unsigned vecWidth) |
| isVEXTRACTIndex - Return true if the specified EXTRACT_SUBVECTOR operand specifies a vector extract that is suitable for instruction that extract 128 or 256 bit vectors More... | |
| static bool | isVINSERTIndex (SDNode *N, unsigned vecWidth) |
| isVINSERTIndex - Return true if the specified INSERT_SUBVECTOR operand specifies a subvector insert that is suitable for input to insertion of 128 or 256-bit subvectors More... | |
| static unsigned | getExtractVEXTRACTImmediate (SDNode *N, unsigned vecWidth) |
| static unsigned | getInsertVINSERTImmediate (SDNode *N, unsigned vecWidth) |
| static bool | isZero (SDValue V) |
| isZero - Returns true if Elt is a constant integer zero More... | |
| static SDValue | getZeroVector (EVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG, SDLoc dl) |
| getZeroVector - Returns a vector of specified type with all zero elements. More... | |
| static SDValue | ExtractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl, unsigned vectorWidth) |
| static SDValue | Extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl) |
| Generate a DAG to grab 128-bits from a vector > 128 bits. More... | |
| static SDValue | Extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl) |
| Generate a DAG to grab 256-bits from a 512-bit vector. More... | |
| static SDValue | InsertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl, unsigned vectorWidth) |
| static SDValue | Insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl) |
| Generate a DAG to put 128-bits into a vector > 128 bits. More... | |
| static SDValue | Insert256BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, SDLoc dl) |
| static SDValue | Concat128BitVectors (SDValue V1, SDValue V2, EVT VT, unsigned NumElems, SelectionDAG &DAG, SDLoc dl) |
| Concat two 128-bit vectors into a 256 bit vector using VINSERTF128 instructions. More... | |
| static SDValue | Concat256BitVectors (SDValue V1, SDValue V2, EVT VT, unsigned NumElems, SelectionDAG &DAG, SDLoc dl) |
| static SDValue | getOnesVector (MVT VT, bool HasInt256, SelectionDAG &DAG, SDLoc dl) |
| getOnesVector - Returns a vector of specified type with all bits set. More... | |
| static SDValue | getUnpackl (SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, SDValue V2) |
| getUnpackl - Returns a vector_shuffle node for an unpackl operation. More... | |
| static SDValue | getUnpackh (SelectionDAG &DAG, SDLoc dl, MVT VT, SDValue V1, SDValue V2) |
| getUnpackh - Returns a vector_shuffle node for an unpackh operation. More... | |
| static SDValue | getShuffleVectorZeroOrUndef (SDValue V2, unsigned Idx, bool IsZero, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified vector of zero or undef vector. More... | |
| static bool | getTargetShuffleMask (SDNode *N, MVT VT, SmallVectorImpl< int > &Mask, bool &IsUnary) |
| getTargetShuffleMask - Calculates the shuffle mask corresponding to the target specific opcode. More... | |
| static SDValue | getShuffleScalarElt (SDNode *N, unsigned Index, SelectionDAG &DAG, unsigned Depth) |
| getShuffleScalarElt - Returns the scalar element that will make up the ith element of the result of the vector shuffle. More... | |
| static SDValue | LowerBuildVectorv16i8 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget *Subtarget, const TargetLowering &TLI) |
| LowerBuildVectorv16i8 - Custom lower build_vector of v16i8. More... | |
| static SDValue | LowerBuildVectorv8i16 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget *Subtarget, const TargetLowering &TLI) |
| LowerBuildVectorv8i16 - Custom lower build_vector of v8i16. More... | |
| static SDValue | LowerBuildVectorv4x32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget, const TargetLowering &TLI) |
| LowerBuildVectorv4x32 - Custom lower build_vector of v4i32 or v4f32. More... | |
| static SDValue | getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, SDLoc dl) |
| Return a vector logical shift node. More... | |
| static SDValue | LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, SDLoc dl, SelectionDAG &DAG) |
| static SDValue | EltsFromConsecutiveLoads (EVT VT, ArrayRef< SDValue > Elts, SDLoc &DL, SelectionDAG &DAG, bool isAfterLegalize) |
| Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'. More... | |
| static SDValue | LowerVectorBroadcast (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| LowerVectorBroadcast - Attempt to use the vbroadcast instruction to generate a splat value for the following cases: More... | |
| static int | getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx) |
| For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index. More... | |
| static SDValue | buildFromShuffleMostly (SDValue Op, SelectionDAG &DAG) |
| static SDValue | ConvertI1VectorToInterger (SDValue Op, SelectionDAG &DAG) |
| static bool | isHorizontalBinOp (const BuildVectorSDNode *N, unsigned Opcode, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1) |
Return true if N implements a horizontal binop and return the operands for the horizontal binop into V0 and V1. More... | |
| static SDValue | ExpandHorizontalBinOp (const SDValue &V0, const SDValue &V1, SDLoc DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI) |
| Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector. More... | |
| static SDValue | LowerToAddSub (const BuildVectorSDNode *BV, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Try to fold a build_vector that performs an 'addsub' to an X86ISD::ADDSUB node. More... | |
| static SDValue | LowerToHorizontalOp (const BuildVectorSDNode *BV, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Lower BUILD_VECTOR to a horizontal add/sub operation if possible. More... | |
| static SDValue | LowerAVXCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerCONCAT_VECTORSvXi1 (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCONCAT_VECTORS (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static bool | isNoopShuffleMask (ArrayRef< int > Mask) |
| Tiny helper function to identify a no-op mask. More... | |
| static bool | isSingleInputShuffleMask (ArrayRef< int > Mask) |
| Helper function to classify a mask as a single-input mask. More... | |
| static bool | is128BitLaneCrossingShuffleMask (MVT VT, ArrayRef< int > Mask) |
| Test whether there are elements crossing 128-bit lanes in this shuffle mask. More... | |
| static bool | is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
| Test whether a shuffle mask is equivalent within each 128-bit lane. More... | |
| static bool | isShuffleEquivalent (SDValue V1, SDValue V2, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask) |
| Checks whether a shuffle mask is equivalent to an explicit list of arguments. More... | |
| static SDValue | getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, SDLoc DL, SelectionDAG &DAG) |
| Get a 4-lane 8-bit shuffle immediate for a mask. More... | |
| static SDValue | lowerVectorShuffleAsBitBlend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Try to emit a blend instruction for a shuffle using bit math. More... | |
| static SDValue | lowerVectorShuffleAsBlend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Try to emit a blend instruction for a shuffle. More... | |
| static SDValue | lowerVectorShuffleAsBlendAndPermute (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Try to lower as a blend of elements from two inputs followed by a single-input permutation. More... | |
| static SDValue | lowerVectorShuffleAsDecomposedShuffleBlend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Generic routine to decompose a shuffle and blend into indepndent blends and permutes. More... | |
| static SDValue | lowerVectorShuffleAsByteRotate (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Try to lower a vector shuffle as a byte rotation. More... | |
| static SmallBitVector | computeZeroableShuffleElements (ArrayRef< int > Mask, SDValue V1, SDValue V2) |
| Compute whether each element of a shuffle is zeroable. More... | |
| static SDValue | lowerVectorShuffleAsBitMask (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Try to emit a bitmask instruction for a shuffle. More... | |
| static SDValue | lowerVectorShuffleAsShift (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Try to lower a vector shuffle as a bit shift (shifts in zeros). More... | |
| static SDValue | lowerVectorShuffleWithSSE4A (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ. More... | |
| static SDValue | lowerVectorShuffleAsSpecificZeroOrAnyExtend (SDLoc DL, MVT VT, int Scale, bool AnyExt, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Lower a vector shuffle as a zero or any extension. More... | |
| static SDValue | lowerVectorShuffleAsZeroOrAnyExtend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Try to lower a vector shuffle as a zero extension on any microarch. More... | |
| static SDValue | getScalarValueForVectorElement (SDValue V, int Idx, SelectionDAG &DAG) |
| Try to get a scalar value for a specific element of a vector. More... | |
| static bool | isShuffleFoldableLoad (SDValue V) |
| Helper to test for a load that can be folded with x86 shuffles. More... | |
| static SDValue | lowerVectorShuffleAsElementInsertion (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Try to lower insertion of a single element into a zero vector. More... | |
| static SDValue | lowerVectorShuffleAsBroadcast (SDLoc DL, MVT VT, SDValue V, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Try to lower broadcast of a single element. More... | |
| static SDValue | lowerVectorShuffleAsInsertPS (SDValue Op, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| static SDValue | lowerVectorShuffleAsUnpack (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction. More... | |
| static SDValue | lowerV2F64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 2-lane 64-bit floating point shuffles. More... | |
| static SDValue | lowerV2I64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 2-lane 64-bit integer shuffles. More... | |
| static bool | isSingleSHUFPSMask (ArrayRef< int > Mask) |
| Test whether this can be lowered with a single SHUFPS instruction. More... | |
| static SDValue | lowerVectorShuffleWithSHUFPS (SDLoc DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| Lower a vector shuffle using the SHUFPS instruction. More... | |
| static SDValue | lowerV4F32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Lower 4-lane 32-bit floating point shuffles. More... | |
| static SDValue | lowerV4I32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Lower 4-lane i32 vector shuffles. More... | |
| static SDValue | lowerV8I16GeneralSingleInputVectorShuffle (SDLoc DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part. More... | |
| static SDValue | lowerVectorShuffleAsPSHUFB (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) |
| Helper to form a PSHUFB-based shuffle+blend. More... | |
| static SDValue | lowerV8I16VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Generic lowering of 8-lane i16 shuffles. More... | |
| static int | canLowerByDroppingEvenElements (ArrayRef< int > Mask) |
| Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped. More... | |
| static SDValue | lowerV16I8VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Generic lowering of v16i8 shuffles. More... | |
| static SDValue | lower128BitVectorShuffle (SDValue Op, SDValue V1, SDValue V2, MVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Dispatching routine to lower various 128-bit x86 vector shuffles. More... | |
| static bool | canWidenShuffleElements (ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask) |
| Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled. More... | |
| static SDValue | splitAndLowerVectorShuffle (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Generic routine to split vector shuffle into half-sized shuffles. More... | |
| static SDValue | lowerVectorShuffleAsSplitOrBlend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Either split a vector in halves or decompose the shuffles and the blend. More... | |
| static SDValue | lowerVectorShuffleAsLanePermuteAndBlend (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Lower a vector shuffle crossing multiple 128-bit lanes as a permutation and blend of those lanes. More... | |
| static SDValue | lowerV2X128VectorShuffle (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering 2-lane 128-bit shuffles. More... | |
| static SDValue | lowerVectorShuffleByMerging128BitLanes (SDLoc DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane. More... | |
| static bool | isShuffleMaskInputInPlace (int Input, ArrayRef< int > Mask) |
| Test whether the specified input (0 or 1) is in-place blended by the given mask. More... | |
| static SDValue | lowerVectorShuffleWithSHUFPD (SDLoc DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| static SDValue | lowerV4F64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 4-lane 64-bit floating point shuffles. More... | |
| static SDValue | lowerV4I64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 4-lane 64-bit integer shuffles. More... | |
| static SDValue | lowerV8F32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 32-bit floating point shuffles. More... | |
| static SDValue | lowerV8I32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 32-bit integer shuffles. More... | |
| static SDValue | lowerV16I16VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 16-lane 16-bit integer shuffles. More... | |
| static SDValue | lowerV32I8VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 32-lane 8-bit integer shuffles. More... | |
| static SDValue | lower256BitVectorShuffle (SDValue Op, SDValue V1, SDValue V2, MVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| High-level routine to lower various 256-bit x86 vector shuffles. More... | |
| static SDValue | lowerV8F64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 64-bit floating point shuffles. More... | |
| static SDValue | lowerV16F32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 16-lane 32-bit floating point shuffles. More... | |
| static SDValue | lowerV8I64VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 64-bit integer shuffles. More... | |
| static SDValue | lowerV16I32VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 16-lane 32-bit integer shuffles. More... | |
| static SDValue | lowerV32I16VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 32-lane 16-bit integer shuffles. More... | |
| static SDValue | lowerV64I8VectorShuffle (SDValue Op, SDValue V1, SDValue V2, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Handle lowering of 64-lane 8-bit integer shuffles. More... | |
| static SDValue | lower512BitVectorShuffle (SDValue Op, SDValue V1, SDValue V2, MVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| High-level routine to lower various 512-bit x86 vector shuffles. More... | |
| static SDValue | lowerVectorShuffle (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Top-level lowering for x86 vector shuffles. More... | |
| static bool | BUILD_VECTORtoBlendMask (BuildVectorSDNode *BuildVector, unsigned &MaskValue) |
| static SDValue | lowerVSELECTtoVectorShuffle (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Try to lower a VSELECT instruction to a vector shuffle. More... | |
| static SDValue | LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerSCALAR_TO_VECTOR (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | GetTLSADDR (SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LocalDynamic=false) |
| static SDValue | LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT) |
| static SDValue | LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT) |
| static SDValue | LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool is64Bit) |
| static SDValue | LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC) |
| static SDValue | LowerShiftParts (SDValue Op, SelectionDAG &DAG) |
| LowerShiftParts - Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount. More... | |
| static SDValue | lowerUINT_TO_FP_vXi32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerAVXExtend (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | LowerZERO_EXTEND_AVX512 (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerANY_EXTEND (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerZERO_EXTEND (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerFP_EXTEND (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerFABSorFNEG (SDValue Op, SelectionDAG &DAG) |
| The only differences between FABS and FNEG are the mask and the logic op. More... | |
| static SDValue | LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerFGETSIGN (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerVectorAllZeroTest (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static bool | hasNonFlagsUse (SDValue Op) |
return true if Op has a use that doesn't just read flags. More... | |
| static bool | isAllOnes (SDValue V) |
| static int | translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1) |
| |
| static SDValue | Lower256IntVSETCC (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerBoolVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerIntVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | ChangeVSETULTtoVSETULE (SDLoc dl, SDValue Op1, SelectionDAG &DAG) |
Try to turn a VSETULT into a VSETULE by modifying its second operand Op1. More... | |
| static SDValue | LowerVSETCC (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static bool | isX86LogicalCmp (SDValue Op) |
| static bool | isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG) |
| static SDValue | LowerSIGN_EXTEND_AVX512 (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerSIGN_EXTEND_VECTOR_INREG (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerSIGN_EXTEND (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerExtendedLoad (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static bool | isAndOrOfSetCCs (SDValue Op, unsigned &Opc) |
| static bool | isXor1OfSetCC (SDValue Op) |
| static SDValue | LowerVACOPY (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | getTargetVShiftByConstNode (unsigned Opc, SDLoc dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG) |
| static SDValue | getTargetVShiftNode (unsigned Opc, SDLoc dl, MVT VT, SDValue SrcOp, SDValue ShAmt, SelectionDAG &DAG) |
| static SDValue | getVectorMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting for Mask when lowering masking intrinsics. More... | |
| static SDValue | getScalarMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Creates an SDNode for a predicated scalar operation. More... | |
| static int | getSEHRegistrationNodeSize (const Function *Fn) |
| static SDValue | recoverFramePointer (SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP) |
| When the 32-bit MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP. More... | |
| static SDValue | LowerINTRINSIC_WO_CHAIN (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | getGatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget *Subtarget) |
| static SDValue | getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain) |
| static SDValue | getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain) |
| static void | getReadPerformanceCounter (SDNode *N, SDLoc DL, SelectionDAG &DAG, const X86Subtarget *Subtarget, SmallVectorImpl< SDValue > &Results) |
| static void | getReadTimeStampCounter (SDNode *N, SDLoc DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget *Subtarget, SmallVectorImpl< SDValue > &Results) |
| static SDValue | LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerSEHRESTOREFRAME (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerCTLZ (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerCTLZ_ZERO_UNDEF (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerCTTZ (SDValue Op, SelectionDAG &DAG) |
| static SDValue | Lower256IntArith (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerADD (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerSUB (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerMUL (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMUL_LOHI (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static bool | SupportedVectorShiftWithImm (MVT VT, const X86Subtarget *Subtarget, unsigned Opcode) |
| static bool | SupportedVectorShiftWithBaseAmnt (MVT VT, const X86Subtarget *Subtarget, unsigned Opcode) |
| static bool | SupportedVectorVarShift (MVT VT, const X86Subtarget *Subtarget, unsigned Opcode) |
| static SDValue | LowerScalarImmediateShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | LowerScalarVariableShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | LowerShift (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerXALUO (SDValue Op, SelectionDAG &DAG) |
| static bool | hasMFENCE (const X86Subtarget &Subtarget) |
| static SDValue | LowerATOMIC_FENCE (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCMP_SWAP (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerBITCAST (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerHorizontalByteSum (SDValue V, MVT VT, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| Compute the horizontal sum of bytes in V for the elements of VT. More... | |
| static SDValue | LowerVectorCTPOPInRegLUT (SDValue Op, SDLoc DL, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerVectorCTPOPBitmath (SDValue Op, SDLoc DL, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerVectorCTPOP (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCTPOP (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerLOAD_SUB (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerADDC_ADDE_SUBC_SUBE (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerFSINCOS (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMSCATTER (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMGATHER (SDValue Op, const X86Subtarget *Subtarget, SelectionDAG &DAG) |
| static MachineBasicBlock * | EmitXBegin (MachineInstr *MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII) |
| Utility function to emit xbegin specifying the start of an RTM region. More... | |
| static MachineBasicBlock * | EmitPCMPSTRM (MachineInstr *MI, MachineBasicBlock *BB, const TargetInstrInfo *TII) |
| static MachineBasicBlock * | EmitPCMPSTRI (MachineInstr *MI, MachineBasicBlock *BB, const TargetInstrInfo *TII) |
| static MachineBasicBlock * | EmitMonitor (MachineInstr *MI, MachineBasicBlock *BB, const X86Subtarget *Subtarget) |
| static bool | checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI) |
| static bool | isShuffleHigh128VectorInsertLow (ShuffleVectorSDNode *SVOp) |
| isShuffleHigh128VectorInsertLow - Checks whether the shuffle node is the same as extracting the high 128-bit part of 256-bit vector and then inserting the result into the low part of a new 256-bit vector More... | |
| static bool | isShuffleLow128VectorInsertHigh (ShuffleVectorSDNode *SVOp) |
| isShuffleLow128VectorInsertHigh - Checks whether the shuffle node is the same as extracting the low 128-bit part of 256-bit vector and then inserting the result into the high part of a new 256-bit vector More... | |
| static SDValue | PerformShuffleCombine256 (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| PerformShuffleCombine256 - Performs shuffle combines for 256-bit vectors. More... | |
| static bool | combineX86ShuffleChain (SDValue Op, SDValue Root, ArrayRef< int > Mask, int Depth, bool HasPSHUFB, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| Combine an arbitrary chain of shuffles into a single instruction if possible. More... | |
| static bool | combineX86ShufflesRecursively (SDValue Op, SDValue Root, ArrayRef< int > RootMask, int Depth, bool HasPSHUFB, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| Fully generic combining of x86 shuffle instructions. More... | |
| static SmallVector< int, 4 > | getPSHUFShuffleMask (SDValue N) |
| Get the PSHUF-style mask from PSHUF node. More... | |
| static SDValue | combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| Search for a combinable shuffle across a chain ending in pshufd. More... | |
| static bool | combineRedundantHalfShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| Search for a combinable shuffle across a chain ending in pshuflw or pshufhw. More... | |
| static SDValue | PerformTargetShuffleCombine (SDValue N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| Try to combine x86 target specific shuffles. More... | |
| static SDValue | combineShuffleToAddSub (SDNode *N, SelectionDAG &DAG) |
| Try to combine a shuffle into a target-specific add-sub node. More... | |
| static SDValue | PerformShuffleCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| PerformShuffleCombine - Performs several different shuffle combines. More... | |
| static SDValue | XFormVExtractWithShuffleIntoLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| XFormVExtractWithShuffleIntoLoad - Check if a vector extract from a target specific shuffle of a load can be folded into a single element load. More... | |
| static SDValue | PerformBITCASTCombine (SDNode *N, SelectionDAG &DAG) |
| Detect bitcasts between i32 to x86mmx low word. More... | |
| static SDValue | PerformEXTRACT_VECTOR_ELTCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| PerformEXTRACT_VECTOR_ELTCombine - Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence. More... | |
| static std::pair< unsigned, bool > | matchIntegerMINMAX (SDValue Cond, EVT VT, SDValue LHS, SDValue RHS, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| Matches a VSELECT onto min/max or return 0 if the node doesn't match. More... | |
| static SDValue | transformVSELECTtoBlendVECTOR_SHUFFLE (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | PerformSELECTCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| PerformSELECTCombine - Do target-specific dag combines on SELECT and VSELECT nodes. More... | |
| static SDValue | checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC) |
| static bool | checkBoolTestAndOrSetCCCombine (SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd) |
| Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS. More... | |
| static SDValue | PerformCMOVCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]. More... | |
| static SDValue | PerformINTRINSIC_WO_CHAINCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | PerformMulCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| PerformMulCombine - Optimize a single multiply with constant into two in order to implement it with two cheaper instructions, e.g. More... | |
| static SDValue | PerformSHLCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performShiftToAllZeros (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| Returns a vector of 0s if the node in input is a vector logical shift by a constant amount which is known to be bigger than or equal to the vector element size in bits. More... | |
| static SDValue | PerformShiftCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| PerformShiftCombine - Combine shifts. More... | |
| static SDValue | CMPEQCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| static bool | CanFoldXORWithAllOnes (const SDNode *N) |
| CanFoldXORWithAllOnes - Test whether the XOR operand is a AllOnes vector so it can be folded inside ANDNP. More... | |
| static SDValue | WidenMaskArithmetic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| static SDValue | VectorZextCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| static SDValue | PerformAndCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| static SDValue | PerformOrCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| static SDValue | performIntegerAbsCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | PerformXorCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| static SDValue | PerformLOADCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| PerformLOADCombine - Do target-specific dag combines on LOAD nodes. More... | |
| static SDValue | PerformMLOADCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| PerformMLOADCombine - Resolve extending loads. More... | |
| static SDValue | PerformMSTORECombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| PerformMSTORECombine - Resolve truncating stores. More... | |
| static SDValue | PerformSTORECombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| PerformSTORECombine - Do target-specific dag combines on STORE nodes. More... | |
| static bool | isHorizontalBinOp (SDValue &LHS, SDValue &RHS, bool IsCommutative) |
| Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS. More... | |
| static SDValue | PerformFADDCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| Do target-specific dag combines on floating point adds. More... | |
| static SDValue | PerformFSUBCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| Do target-specific dag combines on floating point subs. More... | |
| static SDValue | PerformFORCombine (SDNode *N, SelectionDAG &DAG) |
| Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes. More... | |
| static SDValue | PerformFMinFMaxCombine (SDNode *N, SelectionDAG &DAG) |
| Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes. More... | |
| static SDValue | PerformFANDCombine (SDNode *N, SelectionDAG &DAG) |
| Do target-specific dag combines on X86ISD::FAND nodes. More... | |
| static SDValue | PerformFANDNCombine (SDNode *N, SelectionDAG &DAG) |
| Do target-specific dag combines on X86ISD::FANDN nodes. More... | |
| static SDValue | PerformBTCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | PerformVZEXT_MOVLCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | PerformSIGN_EXTEND_INREGCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | PerformSExtCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| static SDValue | PerformFMACombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | PerformZExtCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| static SDValue | PerformISDSETCCCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | NarrowVectorLoadToElement (LoadSDNode *Load, unsigned Index, SelectionDAG &DAG) |
| static SDValue | PerformINSERTPSCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | PerformBLENDICombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | MaterializeSETB (SDLoc DL, SDValue EFLAGS, SelectionDAG &DAG, MVT VT) |
| static SDValue | PerformSETCCCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| static SDValue | PerformBrCondCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| static SDValue | performVectorCompareAndMaskUnaryOpCombine (SDNode *N, SelectionDAG &DAG) |
| static SDValue | PerformUINT_TO_FPCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | PerformSINT_TO_FPCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | PerformADCCombine (SDNode *N, SelectionDAG &DAG, X86TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | OptimizeConditionalInDecrement (SDNode *N, SelectionDAG &DAG) |
| static SDValue | PerformAddCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| PerformADDCombine - Do target-specific dag combines on integer adds. More... | |
| static SDValue | PerformSubCombine (SDNode *N, SelectionDAG &DAG, const X86Subtarget *Subtarget) |
| static SDValue | performVZEXTCombine (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget *Subtarget) |
| performVZEXTCombine - Performs build vector combines More... | |
| static bool | matchAsm (StringRef S, ArrayRef< const char * > Pieces) |
| static bool | clobbersFlagRegisters (const SmallVector< StringRef, 4 > &AsmPieces) |
Variables | |
| static cl::opt< bool > | ExperimentalVectorWideningLegalization ("x86-experimental-vector-widening-legalization", cl::init(false), cl::desc("Enable an experimental vector type legalization through widening ""rather than promotion."), cl::Hidden) |
| #define DEBUG_TYPE "x86-isel" |
Definition at line 60 of file X86ISelLowering.cpp.
| enum StructReturnType |
CallIsStructReturn - Determines whether a call uses struct return semantics.
| Enumerator | |
|---|---|
| NotStructReturn | |
| RegStructReturn | |
| StackStructReturn | |
Definition at line 2241 of file X86ISelLowering.cpp.
|
static |
Determines whether a function uses struct return semantics.
Definition at line 2261 of file X86ISelLowering.cpp.
References llvm::SmallVectorBase::empty(), llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSRet(), NotStructReturn, RegStructReturn, and StackStructReturn.
|
static |
Definition at line 10660 of file X86ISelLowering.cpp.
References llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), and isZero().
Referenced by transformVSELECTtoBlendVECTOR_SHUFFLE().
|
static |
Definition at line 5260 of file X86ISelLowering.cpp.
References llvm::DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), getUnderlyingExtractedFromVec(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::INSERT_VECTOR_ELT, llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::AArch64CC::NV, llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::ISD::UNDEF.
|
static |
Definition at line 2247 of file X86ISelLowering.cpp.
References llvm::SmallVectorBase::empty(), llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSRet(), NotStructReturn, RegStructReturn, and StackStructReturn.
CanFoldXORWithAllOnes - Test whether the XOR operand is a AllOnes vector so it can be folded inside ANDNP.
Definition at line 23429 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is256BitVector(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::UNDEF, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by PerformAndCombine().
Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped.
This handles shuffles which take every Nth element where N is a power of two. Example shuffle masks:
N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24
Any of these lanes can of course be undef.
This routine only supports N <= 3. FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here for larger N.
Definition at line 8854 of file X86ISelLowering.cpp.
References llvm::array_lengthof(), llvm::isPowerOf2_32(), isSingleInputShuffleMask(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I8VectorShuffle().
|
static |
Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.
Appends the mask for wider elements in WidenedMask if valid. Otherwise leaves it in an unspecified state.
NOTE: This must handle normal vector shuffle masks and target vector shuffle masks. The latter have the special property of a '-2' representing a zero-ed lane of a vector.
Definition at line 9235 of file X86ISelLowering.cpp.
References llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineX86ShufflesRecursively(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), and lowerVectorShuffle().
|
static |
Try to turn a VSETULT into a VSETULE by modifying its second operand Op1.
If non-trivial (for example because it's not constant) return an empty value.
Definition at line 13505 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::ConstantSDNode::isOpaque(), and llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back().
Referenced by LowerVSETCC().
|
static |
|
static |
Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
Match: (X86or (X86setcc) (X86setcc)) (X86cmp (and (X86setcc) (X86setcc)), 0)
Definition at line 22827 of file X86ISelLowering.cpp.
References AND, llvm::ARMISD::CMP, llvm::dyn_cast(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::ConstantSDNode::isNullValue(), OR, and llvm::ISD::SETCC.
Referenced by PerformCMOVCombine().
|
static |
Definition at line 22706 of file X86ISelLowering.cpp.
References AND, llvm::ARMISD::CMOV, llvm::ARMISD::CMP, llvm::X86::COND_B, MSP430CC::COND_E, MSP430CC::COND_NE, llvm::CS, llvm::dyn_cast(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), GetOppositeBranchCondition(), llvm::SDValue::getResNo(), llvm::ConstantSDNode::getZExtValue(), llvm::SDNode::hasAnyUseOfValue(), llvm::RDRAND, llvm::RDSEED, llvm::ISD::SETCC, llvm::X86ISD::SETCC_CARRY, SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by PerformBrCondCombine(), PerformCMOVCombine(), and PerformSETCCCombine().
|
static |
Definition at line 25554 of file X86ISelLowering.cpp.
References llvm::SmallVectorTemplateCommon< T >::begin(), llvm::SmallVectorTemplateCommon< T >::end(), and llvm::SmallVectorTemplateCommon< T >::size().
Referenced by llvm::X86TargetLowering::ExpandInlineAsm().
|
static |
Definition at line 23327 of file X86ISelLowering.cpp.
References AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BR_CC, llvm::ISD::BRCOND, llvm::ARMISD::CMP, MSP430CC::COND_E, MSP430CC::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_P, llvm::ISD::CopyToReg, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FSETCC, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::is64Bit(), isAndOrOfSetCCs(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SELECT, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), and llvm::ISD::ZERO_EXTEND.
Referenced by PerformAndCombine(), and PerformOrCombine().
|
static |
Search for a combinable shuffle across a chain ending in pshufd.
We walk up the chain and look for a combinable shuffle, skipping over shuffles that we could hoist this shuffle's transformation past without altering anything.
Definition at line 21284 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::SmallVectorBase::empty(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPSHUFShuffleMask(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::SDNode::isOnlyUserOf(), llvm_unreachable, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by PerformTargetShuffleCombine().
|
static |
Search for a combinable shuffle across a chain ending in pshuflw or pshufhw.
We walk up the chain, skipping shuffles of the other half and looking through shuffles which switch halves trying to find a shuffle of the same pair of dwords.
Definition at line 21417 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPSHUFShuffleMask(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::hasOneUse(), llvm::X86ISD::PSHUFHW, and llvm::X86ISD::PSHUFLW.
Referenced by PerformTargetShuffleCombine().
|
static |
Try to combine a shuffle into a target-specific add-sub node.
We combine this directly on the abstract vector shuffle nodes so it is easier to generically match. We also insert dummy vector shuffle nodes for the operands which explicitly discard the lanes which are unused by this operation to try to flow through the rest of the combiner the fact that they're unused.
Definition at line 21587 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADDSUB, llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), isShuffleEquivalent(), N, llvm::NVPTX::PTXLdStInstCode::V2, and llvm::ISD::VECTOR_SHUFFLE.
Referenced by PerformShuffleCombine().
|
static |
Combine an arbitrary chain of shuffles into a single instruction if possible.
This is the leaf of the recursive combinine below. When we have found some chain of single-use x86 shuffle instructions and accumulated the combined shuffle mask represented by them, this will try to pattern match that mask into either a single instruction if there is a special purpose instruction for this operation, or into a PSHUFB instruction which is a fully general instruction but should only be used to replace chains over a certain depth.
Definition at line 20924 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::DL, llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::equals(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::isFloatingPoint(), and llvm::ArrayRef< T >::size().
Referenced by combineX86ShufflesRecursively().
|
static |
Fully generic combining of x86 shuffle instructions.
This should be the last combine run over the x86 shuffle instructions. Once they have been fully optimized, this will recursively consider all chains of single-use shuffle instructions, build a generic model of the cumulative shuffle operation, and check for simpler instructions which implement this operation. We use this primarily for two purposes:
1) Collapse generic shuffles to specialized single instructions when equivalent. In most cases, this is just an encoding size win, but sometimes we will collapse multiple generic shuffles into a single special-purpose shuffle. 2) Look for sequences of shuffle instructions with 3 or more total instructions, and replace them with the slightly more expensive SSSE3 PSHUFB instruction if available. We do this as the last combining step to ensure we avoid using PSHUFB if we can implement the shuffle with a suitable short sequence of other instructions. The PHUFB will either use a register or have to read from memory and so is slightly (but only slightly) more expensive than the other shuffle instructions.
Because this is inherently a quadratic operation (for each shuffle in a chain, we recurse up the chain), the depth is limited to 8 instructions. This should never be an issue in practice as the shuffle lowering doesn't produce sequences of more than 8 instructions.
FIXME: We will currently miss some cases where the redundant shuffling would simplify under the threshold for PSHUFB formation because of combine-ordering. To fix this, we should do the redundant instruction combining in this recursive walk.
Definition at line 21122 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, canWidenShuffleElements(), llvm::SmallVectorImpl< T >::clear(), combineX86ShuffleChain(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetShuffleMask(), llvm::MVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), llvm::SDNode::isOnlyUserOf(), isTargetShuffle(), llvm::MVT::isVector(), llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SmallVectorImpl< T >::reserve(), llvm::SmallVectorTemplateCommon< T >::size(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by PerformShuffleCombine().
|
static |
Compute whether each element of a shuffle is zeroable.
A "zeroable" vector shuffle element is one which can be lowered to zero. Either it is an undef element in the shuffle mask, the element of the input referenced is undef, or the element of the input referenced is known to be zero. Many x86 shuffles can zero lanes cheaply and we often want to handle as many lanes with this technique as possible to simplify the remaining shuffle.
Definition at line 6796 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::ISD::isBuildVectorAllZeros(), llvm::X86::isZeroNode(), llvm::ArrayRef< T >::size(), llvm::ISD::UNDEF, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerVectorShuffle(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsInsertPS(), lowerVectorShuffleAsPSHUFB(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), and lowerVectorShuffleWithSSE4A().
|
static |
Concat two 128-bit vectors into a 256 bit vector using VINSERTF128 instructions.
This is used because creating CONCAT_VECTOR nodes of BUILD_VECTORS returns a larger BUILD_VECTOR while we're trying to lower large BUILD_VECTORS.
Definition at line 4323 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getUNDEF(), and Insert128BitVector().
Referenced by getOnesVector(), and LowerAVXCONCAT_VECTORS().
|
static |
Definition at line 4330 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getUNDEF(), and Insert256BitVector().
Referenced by LowerAVXCONCAT_VECTORS().
|
static |
Definition at line 5332 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::tgtok::In, llvm::ISD::isBuildVectorOfConstantSDNodes(), and llvm::ISD::UNDEF.
|
static |
Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute.
The copy will be passed as a byval function parameter.
Definition at line 2277 of file X86ISelLowering.cpp.
References llvm::ISD::ArgFlagsTy::getByValAlign(), llvm::ISD::ArgFlagsTy::getByValSize(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMemcpy(), and llvm::MVT::i32.
|
static |
Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.
Example: <load i32 *a, load i32 *a+4, undef, undef> -> zextload a
FIXME: we'd also like to handle the case where the last elements are zero rather than undef via VZEXT_LOAD, but we do not detect that case today. There's even a handy isZeroNode for that purpose.
Definition at line 4959 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasAnyUseOfValue(), llvm::MVT::i64, llvm::SelectionDAG::isConsecutiveLoad(), llvm::MemSDNode::isInvariant(), llvm::ISD::isNON_EXTLoad(), llvm::MemSDNode::isNonTemporal(), llvm::TargetLoweringBase::isOperationLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MemSDNode::isVolatile(), llvm::AArch64DB::LD, llvm::ISD::LOAD, llvm::MVT::Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ArrayRef< T >::size(), llvm::ISD::TokenFactor, llvm::ISD::UNDEF, llvm::SelectionDAG::UpdateNodeOperands(), llvm::MVT::v2i64, and llvm::X86ISD::VZEXT_LOAD.
Referenced by LowerINSERT_SUBVECTOR(), and PerformShuffleCombine().
|
static |
Definition at line 19381 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::addOperand(), llvm::X86::AddrNumOperands, llvm::BuildMI(), llvm::TargetOpcode::COPY, llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::X86Subtarget::getInstrInfo(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), and llvm::X86Subtarget::is64Bit().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 19346 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::addOperand(), llvm::BuildMI(), llvm::TargetOpcode::COPY, llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::MachineInstr::hasOneMemOperand(), llvm::MachineOperand::isImplicit(), llvm::MachineOperand::isReg(), llvm_unreachable, llvm::MachineInstr::memoperands_begin(), llvm::MachineInstr::memoperands_end(), and llvm::MachineInstr::setMemRefs().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 19309 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::addOperand(), llvm::BuildMI(), llvm::TargetOpcode::COPY, llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::MachineInstr::hasOneMemOperand(), llvm::MachineOperand::isImplicit(), llvm::MachineOperand::isReg(), llvm_unreachable, llvm::MachineInstr::memoperands_begin(), llvm::MachineInstr::memoperands_end(), and llvm::MachineInstr::setMemRefs().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
|
static |
Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0).
Definition at line 2804 of file X86ISelLowering.cpp.
References llvm::MachineFrameInfo::CreateFixedObject(), llvm::MachinePointerInfo::getFixedStack(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), and llvm::SelectionDAG::getStore().
|
static |
Utility function to emit xbegin specifying the start of an RTM region.
Definition at line 19251 of file X86ISelLowering.cpp.
References llvm::MachineBasicBlock::addLiveIn(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::TargetOpcode::COPY, llvm::MachineFunction::CreateMachineBasicBlock(), llvm::DL, llvm::MachineBasicBlock::end(), llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineBasicBlock::getBasicBlock(), llvm::MachineInstr::getDebugLoc(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), I, llvm::MachineFunction::insert(), llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
|
static |
Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.
This is a helper function of LowerToHorizontalOp(). This function expects two 256-bit vectors called V0 and V1. At first, each vector is split into two separate 128-bit vectors. Then, the resulting 128-bit vectors are used to implement two horizontal binary operations.
The kind of horizontal binary operation is defined by X86Opcode.
Mode specifies how the 128-bit parts of V0 and V1 are passed in input to the two new horizontal binop. When Mode is set, the first horizontal binop dag node would take as input the lower 128-bit of V0 and the upper 128-bit of V0. The second horizontal binop dag node would take as input the lower 128-bit of V1 and the upper 128-bit of V1. Example: HADD V0_LO, V0_HI HADD V1_LO, V1_HI
Otherwise, the first horizontal binop dag node takes as input the lower 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop dag node takes the upper 128-bit of V0 and the upper 128-bit of V1. Example: HADD V0_LO, V1_LO HADD V0_HI, V1_HI
If isUndefLO is set, then the algorithm propagates UNDEF to the lower 128-bits of the result. If isUndefHI is set, then UNDEF is propagated to the upper 128-bits of the result.
Definition at line 5561 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, Extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::AArch64CC::HI, llvm::EVT::is256BitVector(), llvm::AArch64CC::LO, and llvm::ISD::UNDEF.
Referenced by LowerToHorizontalOp().
|
static |
Generate a DAG to grab 128-bits from a vector > 128 bits.
This sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128 or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4 instructions or a simple subregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering EXTRACT_VECTOR_ELT operations easier.
Definition at line 4220 of file X86ISelLowering.cpp.
References ExtractSubVector(), llvm::SDValue::getValueType(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().
Referenced by ExpandHorizontalBinOp(), Lower256IntArith(), Lower256IntVSETCC(), LowerEXTRACT_SUBVECTOR(), LowerShift(), LowerVectorBroadcast(), LowerVectorCTPOP(), PerformSELECTCombine(), PerformShuffleCombine256(), and PerformSTORECombine().
|
static |
Generate a DAG to grab 256-bits from a 512-bit vector.
Definition at line 4228 of file X86ISelLowering.cpp.
References ExtractSubVector(), llvm::SDValue::getValueType(), and llvm::EVT::is512BitVector().
Referenced by LowerEXTRACT_SUBVECTOR().
|
static |
Definition at line 4181 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::makeArrayRef(), llvm::SDNode::op_begin(), and llvm::ISD::UNDEF.
Referenced by Extract128BitVector(), and Extract256BitVector().
|
static |
Return true if the function is being made into a tailcall target by changing its ABI.
Definition at line 2317 of file X86ISelLowering.cpp.
References IsTailCallConvention().
|
static |
Definition at line 2369 of file X86ISelLowering.cpp.
References llvm::sys::path::begin(), llvm::sys::path::end(), llvm::X86Subtarget::is64Bit(), llvm::X86Subtarget::isCallingConvWin64(), and llvm::makeArrayRef().
|
static |
Definition at line 2387 of file X86ISelLowering.cpp.
References llvm::sys::path::begin(), llvm::sys::path::end(), llvm::MachineFunction::getFunction(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::is64Bit(), llvm::X86Subtarget::isCallingConvWin64(), llvm::makeArrayRef(), llvm::Attribute::NoImplicitFloat, llvm::None, and llvm::X86Subtarget::useSoftFloat().
Definition at line 4058 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), and llvm_unreachable.
Referenced by llvm::X86::getExtractVEXTRACT128Immediate(), and llvm::X86::getExtractVEXTRACT256Immediate().
|
static |
Definition at line 15801 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getRegister(), llvm::ConstantSDNode::getSExtValue(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i1, llvm::MVT::i8, llvm_unreachable, llvm::MVT::Other, and llvm::ISD::UNDEF.
Referenced by LowerINTRINSIC_W_CHAIN().
Definition at line 4073 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), and llvm_unreachable.
Referenced by llvm::X86::getInsertVINSERT128Immediate(), and llvm::X86::getInsertVINSERT256Immediate().
Helper for getByValTypeAlignment to determine the desired ByVal argument alignment.
Definition at line 1760 of file X86ISelLowering.cpp.
Referenced by llvm::X86TargetLowering::getByValTypeAlignment().
getMOVLMask - Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width.
Definition at line 4365 of file X86ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().
|
static |
getOnesVector - Returns a vector of specified type with all bits set.
Always build ones vectors as <4 x i32> or <8 x i32>. For 256-bit types with no AVX2 supprt, use two <4 x i32> inserted in a <8 x i32> appropriately. Then bitcast to their original type, ensuring they get CSE'd.
Definition at line 4341 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, Concat128BitVectors(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i32, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::isVector(), llvm_unreachable, llvm::MVT::v4i32, and llvm::MVT::v8i32.
|
static |
Definition at line 15877 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getRegister(), llvm::ConstantSDNode::getSExtValue(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i1, llvm::MVT::i8, and llvm::MVT::Other.
Referenced by LowerINTRINSIC_W_CHAIN().
|
static |
Get the PSHUF-style mask from PSHUF node.
This is a very minor wrapper around getTargetShuffleMask to easy forming v4 PSHUF-style masks that can be reused with such instructions.
Definition at line 21241 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetShuffleMask(), llvm_unreachable, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, and llvm::X86ISD::PSHUFLW.
Referenced by combineRedundantDWordShuffle(), combineRedundantHalfShuffle(), and PerformTargetShuffleCombine().
|
static |
Definition at line 15902 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_PAIR, llvm::DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::AArch64CC::HI, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::AArch64CC::LO, llvm::ISD::OR, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::X86ISD::RDPMC_DAG, and llvm::ISD::SHL.
Referenced by LowerINTRINSIC_W_CHAIN(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Definition at line 15948 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_PAIR, llvm::DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::AArch64CC::HI, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::AArch64CC::LO, llvm::ISD::OR, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::X86ISD::RDTSCP_DAG, and llvm::ISD::SHL.
Referenced by LowerINTRINSIC_W_CHAIN(), LowerREADCYCLECOUNTER(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Creates an SDNode for a predicated scalar operation.
Mask, Op, PreservedSrc). The mask is comming as MVT::i8 and it should be truncated to MVT::i1 while lowering masking intrinsics. The main difference between ScalarMaskingNode and VectorMaskingNode is using "X86select" instead of "vselect". We just can't create the "vselect" node for a scalar instruction. Definition at line 15255 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getValueType(), getZeroVector(), llvm::MVT::i1, isAllOnes(), llvm::X86ISD::SELECT, llvm::ISD::TRUNCATE, and llvm::ISD::UNDEF.
Referenced by LowerINTRINSIC_WO_CHAIN().
|
static |
Try to get a scalar value for a specific element of a vector.
Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.
Definition at line 7297 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::isVector(), and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by lowerV2F64VectorShuffle(), and lowerVectorShuffleAsElementInsertion().
|
static |
Definition at line 15841 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getRegister(), llvm::ConstantSDNode::getSExtValue(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i1, llvm::MVT::i8, llvm_unreachable, and llvm::MVT::Other.
Referenced by LowerINTRINSIC_W_CHAIN().
Definition at line 15272 of file X86ISelLowering.cpp.
References llvm::classifyEHPersonality(), llvm::Function::getPersonalityFn(), llvm::Function::hasPersonalityFn(), llvm::MSVC_CXX, llvm::MSVC_X86SEH, and llvm::report_fatal_error().
Referenced by LowerSEHRESTOREFRAME(), and recoverFramePointer().
|
static |
getShuffleScalarElt - Returns the scalar element that will make up the ith element of the result of the vector shuffle.
Definition at line 4591 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), isTargetShuffle(), llvm::EVT::isVector(), and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by PerformShuffleCombine().
|
static |
getShuffleVectorZeroOrUndef - Return a vector_shuffle of the specified vector of zero or undef vector.
This produces a shuffle where the low element of V2 is swizzled into the zero/undef vector, landing at element Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
Definition at line 4403 of file X86ISelLowering.cpp.
References llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), and llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back().
Referenced by llvm::X86TargetLowering::BuildFILD(), and getTargetVShiftNode().
|
static |
getTargetShuffleMask - Calculates the shuffle mask corresponding to the target specific opcode.
Returns true if the Mask could be calculated. Sets IsUnary to true if only uses one source. Note that this will set IsUnary for shuffles which use a single input multiple times, and in those cases it will adjust the mask to only have indices within that single input. FIXME: Add support for Decode*Mask functions that return SM_SentinelZero.
Definition at line 4424 of file X86ISelLowering.cpp.
References llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ISD::BITCAST, llvm::X86ISD::BLENDI, llvm::ISD::BUILD_VECTOR, llvm::DecodeBLENDMask(), llvm::DecodeMOVDDUPMask(), llvm::DecodeMOVHLPSMask(), llvm::DecodeMOVLHPSMask(), llvm::DecodeMOVSHDUPMask(), llvm::DecodeMOVSLDUPMask(), llvm::DecodePALIGNRMask(), llvm::DecodePSHUFBMask(), llvm::DecodePSHUFHWMask(), llvm::DecodePSHUFLWMask(), llvm::DecodePSHUFMask(), llvm::DecodeScalarMoveMask(), llvm::DecodeSHUFPMask(), llvm::DecodeUNPCKHMask(), llvm::DecodeUNPCKLMask(), llvm::DecodeVPERM2X128Mask(), llvm::DecodeVPERMMask(), llvm::dyn_cast(), llvm::SmallVectorBase::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::APInt::getLoBits(), llvm::SDValue::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZExtValue(), llvm::EVT::isInteger(), llvm::EVT::isVector(), llvm_unreachable, llvm::APInt::lshr(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPD, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVLPD, llvm::X86ISD::MOVLPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::X86ISD::SHUFP, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::UNDEF, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMILPI, llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.
Referenced by combineX86ShufflesRecursively(), getPSHUFShuffleMask(), getShuffleScalarElt(), and XFormVExtractWithShuffleIntoLoad().
|
static |
Definition at line 3695 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i8, llvm_unreachable, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by llvm::X86TargetLowering::BuildFILD(), and LowerShift().
|
static |
Definition at line 3710 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm_unreachable, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPD, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVLPD, llvm::X86ISD::MOVLPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
|
static |
Definition at line 15082 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::i8, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm_unreachable, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ISD::UNDEF, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.
Referenced by getTargetVShiftNode(), LowerMUL(), and LowerScalarImmediateShift().
|
static |
Definition at line 15159 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getShuffleVectorZeroOrUndef(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), getTargetVShiftByConstNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i64, llvm_unreachable, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerINTRINSIC_WO_CHAIN(), and LowerScalarVariableShift().
|
static |
Definition at line 11452 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getCopyFromReg(), llvm::MachineFunction::getFrameInfo(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::TLSModel::LocalDynamic, llvm::MVT::Other, llvm::MachineFrameInfo::setAdjustsStack(), llvm::MachineFrameInfo::setHasCalls(), llvm::X86ISD::TLSADDR, and llvm::X86ISD::TLSBASEADDR.
Referenced by LowerToTLSGeneralDynamicModel32(), LowerToTLSGeneralDynamicModel64(), and LowerToTLSLocalDynamicModel().
For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.
Modifies ExtractedFromVec to the real vector and returns the real index.
Definition at line 5230 of file X86ISelLowering.cpp.
References llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), and isUndefOrInRange().
Referenced by buildFromShuffleMostly().
getUnpackh - Returns a vector_shuffle node for an unpackh operation.
Definition at line 4388 of file X86ISelLowering.cpp.
References llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().
Referenced by LowerAVXExtend().
getUnpackl - Returns a vector_shuffle node for an unpackl operation.
Definition at line 4376 of file X86ISelLowering.cpp.
References llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().
Referenced by llvm::X86TargetLowering::BuildFILD(), and LowerAVXExtend().
|
static |
Get a 4-lane 8-bit shuffle immediate for a mask.
This helper function produces an 8-bit shuffle immediate corresponding to the ubiquitous shuffle encoding scheme used in x86 instructions for shuffling 4 lanes. It can be used with most of the PSHUF instructions for example.
NB: We rely heavily on "undef" masks preserving the input lane.
Definition at line 6409 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::MVT::i8, and llvm::ArrayRef< T >::size().
Referenced by combineRedundantDWordShuffle(), combineRedundantHalfShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerV8I32VectorShuffle(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), lowerVectorShuffleWithSHUFPS(), and PerformTargetShuffleCombine().
|
static |
Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting for Mask when lowering masking intrinsics.
Definition at line 15213 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::CMPM, llvm::X86ISD::CMPMU, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), getZeroVector(), llvm::MVT::i1, isAllOnes(), llvm::EVT::isSimple(), llvm::X86ISD::PCMPEQM, llvm::X86ISD::PCMPGTM, llvm::ISD::UNDEF, and llvm::ISD::VSELECT.
Referenced by LowerINTRINSIC_W_CHAIN(), and LowerINTRINSIC_WO_CHAIN().
|
static |
Return a vector logical shift node.
Definition at line 4863 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getScalarShiftAmountTy(), llvm::EVT::is128BitVector(), llvm::MVT::v2i64, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.
|
static |
getZeroVector - Returns a vector of specified type with all zero elements.
Definition at line 4134 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::MVT::f32, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i32, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::EVT::isVector(), llvm_unreachable, llvm::MVT::v16i32, llvm::MVT::v4f32, llvm::MVT::v4i32, llvm::MVT::v8f32, and llvm::MVT::v8i32.
Referenced by getGatherNode(), getScalarMaskingNode(), getShuffleVectorZeroOrUndef(), getVectorMaskingNode(), LowerAVXExtend(), LowerBuildVectorv16i8(), LowerBuildVectorv4x32(), LowerBuildVectorv8i16(), LowerCONCAT_VECTORSvXi1(), LowerHorizontalByteSum(), LowerINTRINSIC_WO_CHAIN(), LowerScalarImmediateShift(), LowerShift(), lowerV16I8VectorShuffle(), lowerVectorShuffle(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), LowerVSETCC(), performShiftToAllZeros(), and PerformShuffleCombine256().
hasFPCMov - is there a floating point cmov for the specific X86 condition code.
Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
Definition at line 3904 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, and llvm::X86::COND_P.
Referenced by PerformCMOVCombine().
|
static |
Definition at line 17862 of file X86ISelLowering.cpp.
References llvm::X86Subtarget::hasSSE2(), and llvm::X86Subtarget::is64Bit().
Referenced by LowerATOMIC_FENCE().
return true if Op has a use that doesn't just read flags.
Definition at line 12882 of file X86ISelLowering.cpp.
References llvm::ISD::BRCOND, llvm::SDNode::getOpcode(), llvm::SDNode::use_iterator::getOperandNo(), llvm::SDNode::hasOneUse(), llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().
|
static |
Generate a DAG to put 128-bits into a vector > 128 bits.
This sets things up to match to an AVX VINSERTF128/VINSERTI128 or AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a simple superregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering INSERT_VECTOR_ELT operations easier.
Definition at line 4264 of file X86ISelLowering.cpp.
References llvm::X86ISD::BLENDI, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, InsertSubVector(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::MVT::isFloatingPoint(), llvm::RegState::Undef, llvm::ISD::UNDEF, llvm::MVT::v8f32, and llvm::MVT::v8i32.
Referenced by Concat128BitVectors(), LowerINSERT_SUBVECTOR(), LowerSCALAR_TO_VECTOR(), PerformLOADCombine(), and PerformShuffleCombine256().
|
static |
Definition at line 4313 of file X86ISelLowering.cpp.
References llvm::SDValue::getValueType(), InsertSubVector(), and llvm::EVT::is256BitVector().
Referenced by Concat256BitVectors(), and LowerINSERT_SUBVECTOR().
|
static |
Definition at line 4234 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_SUBVECTOR, and llvm::ISD::UNDEF.
Referenced by Insert128BitVector(), and Insert256BitVector().
Test whether there are elements crossing 128-bit lanes in this shuffle mask.
X86 divides up its shuffles into in-lane and cross-lane shuffle operations and we routinely test for these.
Definition at line 6321 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV8F32VectorShuffle(), and lowerVectorShuffleByMerging128BitLanes().
|
static |
Test whether a shuffle mask is equivalent within each 128-bit lane.
This checks a shuffle mask to see if it is performing the same 128-bit lane-relative shuffle in each 128-bit lane. This trivially implies that it is also not lane-crossing. It may however involve a blend from the same lane of a second vector.
The specific repeated shuffle mask is populated in RepeatedMask, as it is non-trivial to compute in the face of undef lanes. The representation is not suitable for use with existing 128-bit shuffles as it will contain entries from both V1 and V2 inputs to the wider mask.
Definition at line 6342 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), llvm::SmallVectorImpl< T >::resize(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I16VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I32VectorShuffle(), and lowerVectorShuffleAsBlend().
Definition at line 13268 of file X86ISelLowering.cpp.
References llvm::C, llvm::dyn_cast(), and llvm::ConstantSDNode::isAllOnesValue().
Referenced by getScalarMaskingNode(), getVectorMaskingNode(), LowerINTRINSIC_W_CHAIN(), and LowerINTRINSIC_WO_CHAIN().
Definition at line 14536 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::ISD::OR, and llvm::X86ISD::SETCC.
Referenced by CMPEQCombine().
|
static |
Return true if the calling convention is a C calling convention.
Definition at line 2296 of file X86ISelLowering.cpp.
References llvm::CallingConv::C, llvm::CallingConv::X86_64_SysV, and llvm::CallingConv::X86_64_Win64.
|
static |
Return true if N implements a horizontal binop and return the operands for the horizontal binop into V0 and V1.
This is a helper function of LowerToHorizontalOp(). This function checks that the build_vector N in input implements a horizontal operation. Parameter Opcode defines the kind of horizontal operation to match. For example, if Opcode is equal to ISD::ADD, then this function checks if N implements a horizontal arithmetic add; if instead Opcode is equal to ISD::SUB, then this function checks if this is a horizontal arithmetic sub.
This function only analyzes elements of N whose indices are in range [BaseIdx, LastIdx).
Definition at line 5447 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::EVT::isVector(), and llvm::ISD::UNDEF.
Referenced by LowerToHorizontalOp(), PerformAddCombine(), PerformFADDCombine(), PerformFSUBCombine(), and PerformSubCombine().
Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS.
A horizontal operation performs the binary operation on successive elements of its first operand, then on successive elements of its second operand, returning the resulting values in a vector. For example, if A = < float a0, float a1, float a2, float a3 > and B = < float b0, float b1, float b2, float b3 > then the result of doing a horizontal operation on A and B is A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >. In short, LHS and RHS are inspected to see if LHS op RHS is of the form A horizontal-op B, for some already available A and B, and if so then LHS is set to A, RHS to B, and the routine returns 'true'. Note that the binary operation should have the property that if one of the operands is UNDEF then the result is UNDEF.
Definition at line 24372 of file X86ISelLowering.cpp.
References llvm::ARM_PROC::A, llvm::SmallVectorTemplateCommon< T >::begin(), llvm::ArrayRef< T >::begin(), llvm::CallingConv::C, llvm::ArrayRef< T >::end(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::ISD::UNDEF, and llvm::ISD::VECTOR_SHUFFLE.
Tiny helper function to identify a no-op mask.
This is a somewhat boring predicate function. It checks whether the mask array input, which is assumed to be a single-input shuffle mask of the kind used by the X86 shuffle instructions (not a fully general ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an in-place shuffle are 'no-op's.
Definition at line 6294 of file X86ISelLowering.cpp.
References llvm::ArrayRef< T >::size().
Referenced by lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsUnpack(), and PerformTargetShuffleCombine().
|
static |
isSequentialOrUndefInRange - Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size].
or is undef.
Definition at line 3997 of file X86ISelLowering.cpp.
References isUndefOrEqual().
Referenced by lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), and lowerVectorShuffleWithSSE4A().
|
static |
Checks whether a shuffle mask is equivalent to an explicit list of arguments.
This is a fast way to test a shuffle mask against a fixed pattern:
if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }
It returns true if the mask is exactly as wide as the argument list, and each element of the mask is either -1 (signifying undef) or the value given in the argument.
Definition at line 6376 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by combineShuffleToAddSub(), lowerV16F32VectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV2X128VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
Helper to test for a load that can be folded with x86 shuffles.
This is particularly important because the set of instructions varies significantly based on whether the operand is a load or not.
Definition at line 7325 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), and llvm::ISD::isNON_EXTLoad().
Referenced by lowerV2F64VectorShuffle(), lowerVectorShuffleAsBroadcast(), and PerformBLENDICombine().
|
static |
isShuffleHigh128VectorInsertLow - Checks whether the shuffle node is the same as extracting the high 128-bit part of 256-bit vector and then inserting the result into the low part of a new 256-bit vector
Definition at line 20791 of file X86ISelLowering.cpp.
References llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), and isUndefOrEqual().
Referenced by PerformShuffleCombine256().
|
static |
isShuffleLow128VectorInsertHigh - Checks whether the shuffle node is the same as extracting the low 128-bit part of 256-bit vector and then inserting the result into the high part of a new 256-bit vector
Definition at line 20807 of file X86ISelLowering.cpp.
References llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), and isUndefOrEqual().
Referenced by PerformShuffleCombine256().
Test whether the specified input (0 or 1) is in-place blended by the given mask.
This returns true if the elements from a particular input are already in the slot required by the given mask and require no permutation.
Definition at line 9690 of file X86ISelLowering.cpp.
References llvm::ArrayRef< T >::size().
Referenced by lowerV4F64VectorShuffle(), and lowerV4I64VectorShuffle().
Helper function to classify a mask as a single-input mask.
This isn't a generic single-input test because in the vector shuffle lowering we canonicalize single inputs to be the first input operand. This means we can more quickly test for a single input by only checking whether an input from the second operand exists. We also assume that the size of mask corresponds to the size of the input vectors which isn't true in the fully general case.
Definition at line 6309 of file X86ISelLowering.cpp.
Referenced by canLowerByDroppingEvenElements(), lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I32VectorShuffle(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleAsUnpack(), and lowerVectorShuffleByMerging128BitLanes().
Test whether this can be lowered with a single SHUFPS instruction.
This is used to disable more specialized lowerings when the shufps lowering will happen to be efficient.
Definition at line 7904 of file X86ISelLowering.cpp.
References llvm::ArrayRef< T >::size().
Referenced by lowerV4F32VectorShuffle().
|
static |
Return true if the calling convention is one that supports tail call optimization.
Definition at line 2290 of file X86ISelLowering.cpp.
References llvm::CallingConv::Fast, llvm::CallingConv::GHC, and llvm::CallingConv::HiPE.
Referenced by FuncIsMadeTailCallSafe().
Definition at line 3666 of file X86ISelLowering.cpp.
References llvm::X86ISD::BLENDI, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPD, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVLPD, llvm::X86ISD::MOVLPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by combineX86ShufflesRecursively(), getShuffleScalarElt(), PerformShuffleCombine(), and XFormVExtractWithShuffleIntoLoad().
|
static |
Definition at line 13876 of file X86ISelLowering.cpp.
References llvm::tgtok::Bits, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SelectionDAG::MaskedValueIsZero(), and llvm::ISD::TRUNCATE.
isUndefInRange - Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef.
Definition at line 3975 of file X86ISelLowering.cpp.
Referenced by lowerVectorShuffleAsSpecificZeroOrAnyExtend(), and lowerVectorShuffleWithSSE4A().
isUndefOrEqual - Val is either less than zero (undef) or equal to the specified value.
Definition at line 3990 of file X86ISelLowering.cpp.
Referenced by isSequentialOrUndefInRange(), isShuffleHigh128VectorInsertLow(), isShuffleLow128VectorInsertHigh(), and PerformShuffleCombine256().
isUndefOrInRange - Return true if Val is undef or if its value falls within the specified range (L, H].
Definition at line 3984 of file X86ISelLowering.cpp.
References llvm::MipsISD::Hi.
Referenced by getUnderlyingExtractedFromVec().
isVEXTRACTIndex - Return true if the specified EXTRACT_SUBVECTOR operand specifies a vector extract that is suitable for instruction that extract 128 or 256 bit vectors
Definition at line 4008 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), and llvm::MVT::getVectorElementType().
Referenced by llvm::X86::isVEXTRACT128Index(), and llvm::X86::isVEXTRACT256Index().
isVINSERTIndex - Return true if the specified INSERT_SUBVECTOR operand specifies a subvector insert that is suitable for input to insertion of 128 or 256-bit subvectors
Definition at line 4027 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), and llvm::MVT::getVectorElementType().
Referenced by llvm::X86::isVINSERT128Index(), and llvm::X86::isVINSERT256Index().
Return true if the condition is an unsigned comparison operation.
Definition at line 3796 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, and llvm_unreachable.
Definition at line 13851 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::X86ISD::ADD, llvm::X86ISD::AND, llvm::X86ISD::CMP, llvm::X86ISD::COMI, llvm::X86ISD::DEC, llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getResNo(), llvm::X86ISD::INC, llvm::X86ISD::OR, llvm::X86ISD::SAHF, llvm::X86ISD::SBB, llvm::X86ISD::SMUL, llvm::X86ISD::SUB, llvm::X86ISD::UCOMI, llvm::X86ISD::UMUL, and llvm::X86ISD::XOR.
Definition at line 14548 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::X86ISD::SETCC, and llvm::ISD::XOR.
isZero - Returns true if Elt is a constant integer zero
Definition at line 4117 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), and llvm::ConstantSDNode::isNullValue().
Referenced by BUILD_VECTORtoBlendMask(), llvm::X86::isZeroNode(), and lowerVSELECTtoVectorShuffle().
|
static |
Dispatching routine to lower various 128-bit x86 vector shuffles.
This routine breaks down the specific type of 128-bit shuffle and dispatches to the lowering routines accordingly.
Definition at line 9204 of file X86ISelLowering.cpp.
References llvm_unreachable, lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4I32VectorShuffle(), lowerV8I16VectorShuffle(), llvm::MVT::SimpleTy, llvm::MVT::v16i8, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4i32, and llvm::MVT::v8i16.
Referenced by lowerVectorShuffle().
|
static |
High-level routine to lower various 256-bit x86 vector shuffles.
This routine either breaks down the specific type of a 256-bit x86 vector shuffle or splits it into two 128-bit shuffles and fuses the results back together based on the available instructions.
Definition at line 10280 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getFloatingPointVT(), llvm::ShuffleVectorSDNode::getMask(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::isInteger(), llvm_unreachable, lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I32VectorShuffle(), lowerVectorShuffleAsElementInsertion(), llvm::MVT::SimpleTy, splitAndLowerVectorShuffle(), llvm::MVT::v16i16, llvm::MVT::v32i8, llvm::MVT::v4f64, llvm::MVT::v4i64, llvm::MVT::v8f32, and llvm::MVT::v8i32.
Referenced by lowerVectorShuffle().
|
static |
Definition at line 16680 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, Extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is256BitVector(), and llvm::MVT::isInteger().
Referenced by LowerADD(), LowerMUL(), and LowerSUB().
|
static |
Definition at line 13391 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, Extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is256BitVector(), and llvm::ISD::SETCC.
Referenced by LowerVSETCC().
|
static |
High-level routine to lower various 512-bit x86 vector shuffles.
This routine either breaks down the specific type of a 512-bit x86 vector shuffle or splits it into two 256-bit shuffles and fuses the results back together based on the available instructions.
Definition at line 10476 of file X86ISelLowering.cpp.
References llvm::DL, llvm::ShuffleVectorSDNode::getMask(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm_unreachable, lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV32I16VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I64VectorShuffle(), lowerVectorShuffleAsBroadcast(), llvm::MVT::SimpleTy, splitAndLowerVectorShuffle(), llvm::MVT::v16f32, llvm::MVT::v16i32, llvm::MVT::v32i16, llvm::MVT::v64i8, llvm::MVT::v8f64, and llvm::MVT::v8i64.
Referenced by lowerVectorShuffle().
|
static |
Definition at line 16707 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), llvm::MVT::i1, llvm::MVT::is256BitVector(), llvm::MVT::isInteger(), Lower256IntArith(), and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 18330 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::X86ISD::ADD, llvm::ISD::ADDC, llvm::ISD::ADDE, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVTList(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, llvm::X86ISD::SBB, llvm::X86ISD::SUB, llvm::ISD::SUBC, and llvm::ISD::SUBE.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 16375 of file X86ISelLowering.cpp.
References llvm::SDValue::getOperand().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 12401 of file X86ISelLowering.cpp.
References llvm::X86Subtarget::hasFp256(), and LowerAVXExtend().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 4877 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::DL, llvm::MVT::f32, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFrameInfo(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::i32, llvm::SelectionDAG::InferPtrAlignment(), llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::MachineFrameInfo::isFixedObjectIndex(), llvm::ISD::isNormalLoad(), llvm::AArch64DB::LD, and llvm::MachineFrameInfo::setObjectAlignment().
|
static |
Definition at line 17928 of file X86ISelLowering.cpp.
References llvm::CrossThread, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), hasMFENCE(), llvm::MVT::i8, llvm::X86ISD::MEMBARRIER, llvm::X86ISD::MFENCE, llvm::MVT::Other, and llvm::SequentiallyConsistent.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 18305 of file X86ISelLowering.cpp.
References llvm::ISD::ATOMIC_SWAP, llvm::SelectionDAG::getAtomic(), llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::SequentiallyConsistent.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 6182 of file X86ISelLowering.cpp.
References Concat128BitVectors(), Concat256BitVectors(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::NVPTX::PTXLdStInstCode::V2, and llvm::NVPTX::PTXLdStInstCode::V4.
Referenced by LowerCONCAT_VECTORS().
|
static |
Definition at line 12329 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasInt256(), llvm::MVT::i1, llvm::MVT::is512BitVector(), llvm::RegState::Undef, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8i16, llvm::MVT::v8i32, llvm::X86ISD::VZEXT, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerANY_EXTEND(), and LowerZERO_EXTEND().
|
static |
Definition at line 18003 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::append(), llvm::ISD::BUILD_VECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasMMX(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i64, llvm::X86Subtarget::is64Bit(), llvm::MVT::isVector(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::MVT::v2f64, llvm::MVT::v2i32, llvm::MVT::v4i16, and llvm::MVT::v8i8.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 13419 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::i1, llvm_unreachable, llvm::ISD::OR, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, and llvm::ISD::XOR.
Referenced by LowerVSETCC().
|
static |
LowerBuildVectorv16i8 - Custom lower build_vector of v16i8.
Definition at line 4655 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i8, llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::OR, llvm::ISD::SHL, llvm::MVT::v16i8, llvm::MVT::v8i16, and llvm::ISD::ZERO_EXTEND.
|
static |
LowerBuildVectorv4x32 - Custom lower build_vector of v4i32 or v4f32.
Definition at line 4757 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::X86::isZeroNode(), llvm::ISD::UNDEF, llvm::NVPTX::PTXLdStInstCode::V2, and llvm::MVT::v4f32.
|
static |
LowerBuildVectorv8i16 - Custom lower build_vector of v8i16.
Definition at line 4726 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), getZeroVector(), llvm::ISD::INSERT_VECTOR_ELT, and llvm::MVT::v8i16.
|
static |
Definition at line 17961 of file X86ISelLowering.cpp.
References llvm::X86::COND_E, llvm::DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::X86ISD::LCMPXCHG_DAG, llvm_unreachable, llvm::MVT::Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::X86ISD::SETCC, llvm::MVT::SimpleTy, size, llvm::Success, and T.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 6257 of file X86ISelLowering.cpp.
References llvm::SDValue::getNumOperands(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::i1, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerAVXCONCAT_VECTORS(), and LowerCONCAT_VECTORSvXi1().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 6206 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MipsISD::Hi, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::isPowerOf2_32(), llvm::MipsISD::Lo, llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::RegState::Undef, llvm::NVPTX::PTXLdStInstCode::V2, llvm::X86ISD::VSHLI, and llvm::X86ISD::VSRLI.
Referenced by LowerCONCAT_VECTORS().
|
static |
Definition at line 16597 of file X86ISelLowering.cpp.
References llvm::X86ISD::BSR, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::MVT::i8, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 16632 of file X86ISelLowering.cpp.
References llvm::X86ISD::BSR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::MVT::i8, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 18283 of file X86ISelLowering.cpp.
References llvm::SDValue::getValueType(), llvm::EVT::isVector(), and LowerVectorCTPOP().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 16658 of file X86ISelLowering.cpp.
References llvm::X86ISD::BSF, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm::MVT::i8.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 14336 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::EXTLOAD, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getExtLoad(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::TargetLoweringBase::getPointerTy(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i8, llvm::ISD::INSERT_VECTOR_ELT, llvm::MVT::integer_valuetypes(), llvm::MVT::isInteger(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::MVT::isVector(), llvm::SPII::Load, llvm::MVT::Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SEXTLOAD, llvm::ISD::SRA, llvm::ISD::TokenFactor, and llvm::X86ISD::VSEXT.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 11153 of file X86ISelLowering.cpp.
References Extract128BitVector(), Extract256BitVector(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasFp256(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), and llvm::MVT::is512BitVector().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 10763 of file X86ISelLowering.cpp.
References Assert, llvm::ISD::AssertZext, llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f32, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::is128BitVector(), llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, llvm::ISD::STORE, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), and llvm::MVT::v4i32.
|
static |
The only differences between FABS and FNEG are the mask and the logic op.
FNEG also has a folding opportunity for FNEG(FABS(x)).
Definition at line 12630 of file X86ISelLowering.cpp.
References llvm::C, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::ISD::FABS, llvm::X86ISD::FAND, llvm::ISD::FNEG, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::ConstantInt::get(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::APInt::getSignBit(), llvm::APInt::getSignedMaxValue(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::ConstantVector::getSplat(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SDNode::uses(), llvm::MVT::v2f64, and llvm::MVT::v4f32.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 12699 of file X86ISelLowering.cpp.
References llvm::lltok::APFloat, llvm::MVT::bitsGT(), llvm::MVT::bitsLT(), llvm::C, llvm::APFloat::clearSign(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::ConstantFP::get(), llvm::ConstantVector::get(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::APFloat::IEEEdouble, llvm::APFloat::IEEEsingle, llvm::APFloat::isPosZero(), llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v2f64, and llvm::MVT::v4f32.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 12778 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::FGETSIGNx86, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), and llvm::SDValue::getSimpleValueType().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 12615 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::v2f32, llvm::MVT::v4f32, and llvm::X86ISD::VFPEXT.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 18356 of file X86ISelLowering.cpp.
References llvm::CallingConv::C, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::StructType::get(), llvm::VectorType::get(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getExternalSymbol(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getTypeForEVT(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::is64Bit(), llvm::TargetLowering::ArgListEntry::isSExt, llvm::X86Subtarget::isTargetDarwin(), llvm::TargetLowering::ArgListEntry::isZExt, llvm::TargetLowering::LowerCallTo(), llvm::ISD::MERGE_VALUES, llvm::TargetLowering::ArgListEntry::Node, llvm::TargetLowering::CallLoweringInfo::setChain(), llvm::TargetLowering::CallLoweringInfo::setDebugLoc(), and llvm::TargetLowering::ArgListEntry::Ty.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Compute the horizontal sum of bytes in V for the elements of VT.
Requires V to be a byte vector and VT to be an integer vector type with wider elements than V's type. The width of the elements of VT determines how many bytes of V are summed horizontally to produce each element of the result.
Definition at line 18059 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::BUILD_VECTOR, llvm::DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::i64, llvm::MVT::i8, llvm::X86ISD::PACKUS, llvm::X86ISD::PSADBW, llvm::ISD::SHL, llvm::ISD::SRL, llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by LowerVectorCTPOPBitmath(), and LowerVectorCTPOPInRegLUT().
|
static |
Definition at line 11179 of file X86ISelLowering.cpp.
References EltsFromConsecutiveLoads(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX(), llvm::MVT::i1, llvm::MVT::i8, Insert128BitVector(), Insert256BitVector(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::X86Subtarget::isUnalignedMem32Slow(), llvm::ISD::OR, llvm::RegState::Undef, llvm::ISD::UNDEF, llvm::X86ISD::VSHLI, and llvm::X86ISD::VSRLI.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 16068 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::ADX, llvm::X86ISD::CMOV, llvm::COMPRESS_TO_MEM, llvm::X86::COND_B, llvm::X86::COND_NE, llvm::MachineFrameInfo::CreateFixedObject(), llvm::EXPAND_FROM_MEM, llvm::GATHER, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::X86MachineFunctionInfo::getFAIndex(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), getGatherNode(), llvm::MachineFunction::getInfo(), llvm::getIntrinsicWithChain(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::TargetMachine::getMCAsmInfo(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), getPrefetchNode(), getReadPerformanceCounter(), getReadTimeStampCounter(), llvm::X86Subtarget::getRegisterInfo(), llvm::X86TargetLowering::getReturnAddressFrameIndex(), llvm::EVT::getScalarSizeInBits(), getScatterNode(), llvm::X86RegisterInfo::getSlotSize(), llvm::SelectionDAG::getStore(), llvm::MachineFunction::getTarget(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), getVectorMaskingNode(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::Glue, Hint(), llvm::MVT::i64, llvm::MVT::i8, isAllOnes(), llvm_unreachable, LowerSEHRESTOREFRAME(), llvm::ISD::MERGE_VALUES, llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, llvm::MVT::Other, llvm::PREFETCH, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::RDPMC, llvm::RDRAND, llvm::RDSEED, llvm::RDTSC, llvm::MipsISD::Ret, llvm::SCATTER, llvm::X86ISD::SETCC, llvm::X86MachineFunctionInfo::setFAIndex(), llvm::MachineFrameInfo::setFrameAddressIsTaken(), llvm::MachineFrameInfo::setReturnAddressIsTaken(), llvm::IntrinsicData::Type, llvm::MCAsmInfo::usesWindowsCFI(), llvm::TargetLowering::verifyReturnAddressArgumentIsConstant(), llvm::XTEST, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 15326 of file X86ISelLowering.cpp.
References llvm::BLEND, llvm::CMP_MASK, llvm::CMP_MASK_CC, llvm::COMI, llvm::COMPRESS_EXPAND_IN_REG, llvm::X86::COND_A, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_INVALID, llvm::X86::COND_O, llvm::X86::COND_S, llvm::X86::CUR_DIRECTION, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::FMA_OP_MASK, llvm::FMA_OP_MASK3, llvm::FMA_OP_MASKZ, llvm::X86RegisterInfo::getBaseRegister(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MachineModuleInfo::getContext(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::getIntrinsicWithoutChain(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMCSymbol(), llvm::MachineFunction::getMMI(), llvm::Value::getName(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::MCContext::getOrCreateLSDASymbol(), llvm::X86RegisterInfo::getPtrSizedFrameRegister(), llvm::GlobalValue::getRealLinkageName(), llvm::X86Subtarget::getRegisterInfo(), getScalarMaskingNode(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), getVectorMaskingNode(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::X86RegisterInfo::hasBasePointer(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::INTR_TYPE_1OP, llvm::INTR_TYPE_1OP_MASK, llvm::INTR_TYPE_1OP_MASK_RM, llvm::INTR_TYPE_2OP, llvm::INTR_TYPE_2OP_MASK, llvm::INTR_TYPE_2OP_MASK_RM, llvm::INTR_TYPE_3OP, llvm::INTR_TYPE_3OP_MASK, llvm::INTR_TYPE_4OP, llvm::INTR_TYPE_SCALAR_MASK_RM, isAllOnes(), llvm::X86ISD::KORTEST, llvm_unreachable, llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, llvm::X86ISD::PCMPESTRI, llvm::X86ISD::PCMPISTRI, llvm::X86ISD::PTEST, recoverFramePointer(), llvm::report_fatal_error(), llvm::X86ISD::SETCC, llvm::X86ISD::TESTP, TranslateX86CC(), llvm::ISD::TRUNCATE, llvm::IntrinsicData::Type, llvm::MVT::v16i1, llvm::VPERM_3OP_MASK, llvm::VPERM_3OP_MASKZ, llvm::X86ISD::VPERMV, llvm::VSHIFT, llvm::VSHIFT_MASK, llvm::X86ISD::Wrapper, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 13462 of file X86ISelLowering.cpp.
References llvm::X86ISD::CMPM, llvm::X86ISD::CMPMU, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::i1, llvm::MVT::i8, llvm_unreachable, llvm::X86ISD::PCMPEQM, llvm::X86ISD::PCMPGTM, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, std::swap(), and Unsigned.
Referenced by LowerVSETCC().
|
static |
Definition at line 18290 of file X86ISelLowering.cpp.
References llvm::ISD::ATOMIC_LOAD_ADD, llvm::SelectionDAG::getAtomic(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), and llvm::ISD::SUB.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 18438 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasVLX(), llvm::EVT::is512BitVector(), llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::UpdateNodeOperands(), and llvm::MVT::v8i64.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 18409 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getMaskedScatter(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasVLX(), llvm::EVT::is512BitVector(), llvm::MVT::Other, llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SIGN_EXTEND, and llvm::MVT::v8i64.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 16727 of file X86ISelLowering.cpp.
References llvm::ARM_PROC::A, llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), getTargetVShiftByConstNode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i8, llvm::MVT::is256BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::MipsISD::Lo, Lower256IntArith(), llvm::ISD::MUL, llvm::X86ISD::PACKUS, llvm::X86ISD::PMULUDQ, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::TRUNCATE, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8i16, llvm::MVT::v8i32, llvm::MVT::v8i64, llvm::X86ISD::VSEXT, llvm::X86ISD::VSHLI, and llvm::X86ISD::VSRLI.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 16933 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, Fixup, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getShiftAmountTy(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86ISD::PMULDQ, llvm::X86ISD::PMULUDQ, llvm::ISD::SMUL_LOHI, llvm::ISD::SRA, llvm::ISD::SUB, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v4i64, and llvm::MVT::v8i32.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 15999 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SDValue::getNode(), getReadTimeStampCounter(), and llvm::X86ISD::RDTSC_DAG.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 11122 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i32, llvm::MVT::i64, Insert128BitVector(), llvm::MVT::is128BitVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v1i64, and llvm::MVT::v4i32.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 17055 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), getTargetVShiftByConstNode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::ConstantSDNode::getZExtValue(), llvm::X86Subtarget::hasInt256(), llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm_unreachable, llvm::Log2_32_Ceil(), llvm::X86ISD::PCMPGT, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, SupportedVectorShiftWithImm(), llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i64, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and llvm::ISD::XOR.
Referenced by LowerShift().
|
static |
Definition at line 17204 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::bitsGT(), llvm::EVT::bitsLE(), llvm::EVT::bitsLT(), llvm::ISD::BUILD_VECTOR, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::ShuffleVectorSDNode::getSplatIndex(), getTargetVShiftNode(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), llvm::MVT::i64, llvm::ISD::INSERT_VECTOR_ELT, llvm::X86Subtarget::is64Bit(), llvm::ShuffleVectorSDNode::isSplat(), llvm::ISD::SHL, llvm::ISD::SRL, SupportedVectorShiftWithBaseAmnt(), llvm::ISD::UNDEF, llvm::MVT::v2i64, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerShift().
|
static |
Definition at line 16008 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::MachineFrameInfo::CreateSpillStackObject(), llvm::X86RegisterInfo::getBaseRegister(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), llvm::X86Subtarget::getFrameLowering(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getInfo(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::X86RegisterInfo::getPtrSizedFrameRegister(), llvm::X86Subtarget::getRegisterInfo(), llvm::MVT::getScalarSizeInBits(), getSEHRegistrationNodeSize(), llvm::X86RegisterInfo::getSlotSize(), llvm::X86RegisterInfo::getStackRegister(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86RegisterInfo::hasBasePointer(), llvm::X86FrameLowering::hasFP(), llvm::X86RegisterInfo::needsStackRealignment(), recoverFramePointer(), llvm::X86MachineFunctionInfo::setHasSEHFramePtrSave(), and llvm::X86MachineFunctionInfo::setSEHFramePtrSaveIndex().
Referenced by LowerINTRINSIC_W_CHAIN().
|
static |
Definition at line 17287 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::ANY_EXTEND, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, Extract128BitVector(), llvm::ISD::FP_TO_SINT, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), getTargetShuffleNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::i8, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MVT::isVector(), llvm_unreachable, llvm::MipsISD::Lo, LowerScalarImmediateShift(), LowerScalarVariableShift(), llvm::makeArrayRef(), llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, llvm::ISD::MUL, One, llvm::SDNode::op_begin(), llvm::ISD::OR, llvm::X86ISD::PACKUS, llvm::X86ISD::PCMPGT, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::R13, R2, llvm::ISD::SHL, llvm::APInt::shl(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::SRL, SupportedVectorVarShift(), llvm::ISD::TRUNCATE, llvm::ISD::UNDEF, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::MVT::v8i32, llvm::ISD::VSELECT, llvm::X86ISD::VSHL, llvm::X86ISD::VSRA, llvm::X86ISD::VSRL, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
LowerShiftParts - Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount.
Definition at line 11743 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::CMOV, llvm::X86ISD::CMP, llvm::X86::COND_NE, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::MVT::i8, llvm::MipsISD::Lo, llvm::ISD::SHL, llvm::ISD::SHL_PARTS, llvm::X86ISD::SHLD, llvm::X86ISD::SHRD, llvm::ISD::SRA, llvm::ISD::SRA_PARTS, and llvm::ISD::SRL.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 14278 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::MVT::i1, llvm::MVT::is512BitVector(), LowerSIGN_EXTEND_AVX512(), llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8i16, llvm::MVT::v8i32, and llvm::X86ISD::VSEXT.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 14172 of file X86ISelLowering.cpp.
References llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::APInt::getNullValue(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::is512BitVector(), llvm::MVT::v16i32, llvm::MVT::v8i64, llvm::ISD::VSELECT, llvm::X86ISD::VSEXT, llvm::X86ISD::VTRUNC, and llvm::X86ISD::VZEXT.
Referenced by LowerSIGN_EXTEND().
|
static |
Definition at line 14222 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::tgtok::In, llvm::X86ISD::UNPCKL, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::X86ISD::VSEXT, and llvm::X86ISD::VSRAI.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 16717 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), llvm::MVT::i1, llvm::MVT::is256BitVector(), llvm::MVT::isInteger(), Lower256IntArith(), and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Try to fold a build_vector that performs an 'addsub' to an X86ISD::ADDSUB node.
Definition at line 5601 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADDSUB, llvm::DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE3(), std::swap(), llvm::ISD::UNDEF, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, and llvm::MVT::v8f32.
|
static |
Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
Definition at line 5703 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::DL, ExpandHorizontalBinOp(), llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, isHorizontalBinOp(), llvm::ISD::SUB, llvm::ISD::UNDEF, llvm::MVT::v16i16, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::MVT::v8f32, llvm::MVT::v8i16, and llvm::MVT::v8i32.
|
static |
Definition at line 11544 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getEntryNode(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachinePointerInfo::getGOT(), llvm::Type::getInt8PtrTy(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SDNode::getValueType(), llvm::X86ISD::GlobalBaseReg, llvm::TLSModel::InitialExec, llvm_unreachable, llvm::TLSModel::LocalExec, llvm::X86II::MO_GOTNTPOFF, llvm::X86II::MO_GOTTPOFF, llvm::X86II::MO_INDNTPOFF, llvm::X86II::MO_NTPOFF, llvm::X86II::MO_TPOFF, llvm::MipsISD::ThreadPointer, llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.
|
static |
Definition at line 11484 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), GetTLSADDR(), llvm::SDValue::getValue(), llvm::X86ISD::GlobalBaseReg, and llvm::X86II::MO_TLSGD.
|
static |
Definition at line 11498 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getEntryNode(), GetTLSADDR(), and llvm::X86II::MO_TLSGD.
|
static |
Definition at line 11504 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getEntryNode(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachineFunction::getInfo(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), GetTLSADDR(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::X86ISD::GlobalBaseReg, llvm::X86MachineFunctionInfo::incNumLocalDynamicTLSAccesses(), llvm::X86II::MO_DTPOFF, llvm::X86II::MO_TLSLD, llvm::X86II::MO_TLSLDM, and llvm::X86ISD::Wrapper.
|
static |
Definition at line 12014 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::lltok::APFloat, llvm::X86ISD::BLENDI, llvm::ISD::BUILD_VECTOR, llvm::DL, llvm::MVT::f32, llvm::ISD::FADD, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i32, llvm::APFloat::IEEEsingle, llvm::makeArrayRef(), llvm::ISD::OR, llvm::ISD::SRL, llvm::MVT::v16i16, llvm::MVT::v4f32, llvm::MVT::v4i32, llvm::MVT::v8f32, llvm::MVT::v8i16, and llvm::MVT::v8i32.
|
static |
Handle lowering of 16-lane 32-bit floating point shuffles.
Definition at line 10360 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), isShuffleEquivalent(), splitAndLowerVectorShuffle(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::MVT::v16f32.
Referenced by lower512BitVectorShuffle().
|
static |
Handle lowering of 16-lane 16-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v16i16 shuffling..
Definition at line 10090 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i8, is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), isSingleInputShuffleMask(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleByMerging128BitLanes(), llvm::X86ISD::PSHUFB, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i16, and llvm::MVT::v32i8.
Referenced by lower256BitVectorShuffle().
|
static |
Handle lowering of 16-lane 32-bit integer shuffles.
Definition at line 10411 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), isShuffleEquivalent(), splitAndLowerVectorShuffle(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::MVT::v16i32.
Referenced by lower512BitVectorShuffle().
|
static |
Generic lowering of v16i8 shuffles.
This is a hybrid strategy to lower v16i8 vectors. It first attempts to detect any complexity reducing interleaving. If that doesn't help, it uses UNPCK to spread the i8 elements across two i16-element vectors, and uses the existing lowering for v8i16 blends on each half, finally PACK-ing them back together.
Definition at line 8908 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::sys::path::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), canLowerByDroppingEvenElements(), llvm::DL, llvm::sys::path::end(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), I, isShuffleEquivalent(), isSingleInputShuffleMask(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPSHUFB(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsUnpack(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), llvm::X86ISD::PACKUS, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, and llvm::MVT::v8i16.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering of 2-lane 64-bit floating point shuffles.
This is the basis function for the 2-lane 64-bit shuffles as we have full support for floating point shuffles but not integer shuffles. These instructions will incur a domain crossing penalty on some chips though so it is better to avoid lowering through this for integer vectors where possible.
Definition at line 7711 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getScalarValueForVectorElement(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i8, isShuffleEquivalent(), isShuffleFoldableLoad(), isSingleInputShuffleMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsElementInsertion(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVLPD, llvm::X86ISD::MOVSD, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v2f64, and llvm::X86ISD::VPERMILPI.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering of 2-lane 64-bit integer shuffles.
Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by the integer unit to minimize domain crossing penalties. However, for blends it falls back to the floating point shuffle operation with appropriate bit casting.
Definition at line 7793 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), isShuffleEquivalent(), isSingleInputShuffleMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsShift(), llvm::X86ISD::PACKUS, llvm::X86ISD::PSHUFD, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i8, llvm::MVT::v2f64, llvm::MVT::v2i64, and llvm::MVT::v4i32.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering 2-lane 128-bit shuffles.
Definition at line 9521 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i8, llvm::ISD::isBuildVectorAllZeros(), isShuffleEquivalent(), lowerVectorShuffleAsBlend(), llvm::SM_SentinelUndef, and llvm::X86ISD::VPERM2X128.
Referenced by lowerV4F64VectorShuffle(), and lowerV4I64VectorShuffle().
|
static |
Handle lowering of 32-lane 16-bit integer shuffles.
Definition at line 10440 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasBWI(), splitAndLowerVectorShuffle(), and llvm::MVT::v32i16.
Referenced by lower512BitVectorShuffle().
|
static |
Handle lowering of 32-lane 8-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v32i8 shuffling..
Definition at line 10190 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i8, is128BitLaneCrossingShuffleMask(), isShuffleEquivalent(), isSingleInputShuffleMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleByMerging128BitLanes(), llvm::X86ISD::PSHUFB, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::MVT::v32i8.
Referenced by lower256BitVectorShuffle().
|
static |
Lower 4-lane 32-bit floating point shuffles.
Uses instructions exclusively from the floating point unit to minimize domain crossing penalties, as these are sufficient to implement all v4f32 shuffles.
Definition at line 8013 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), isShuffleEquivalent(), isSingleSHUFPSMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBlendAndPermute(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsInsertPS(), lowerVectorShuffleWithSHUFPS(), llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v4f32, and llvm::X86ISD::VPERMILPI.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering of 4-lane 64-bit floating point shuffles.
Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2 isn't available.
Definition at line 9735 of file X86ISelLowering.cpp.
References canWidenShuffleElements(), llvm::DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i8, is128BitLaneCrossingShuffleMask(), isShuffleEquivalent(), isShuffleMaskInputInPlace(), isSingleInputShuffleMask(), lowerV2X128VectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleWithSHUFPD(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v4f64, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by lower256BitVectorShuffle().
|
static |
Lower 4-lane i32 vector shuffles.
We try to handle these with integer-domain shuffles where we can, but for blends we use the floating point domain blend instructions.
Definition at line 8097 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), isShuffleEquivalent(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsUnpack(), lowerVectorShuffleAsZeroOrAnyExtend(), llvm::X86ISD::PSHUFD, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v4f32, and llvm::MVT::v4i32.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering of 4-lane 64-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v4i64 shuffling..
Definition at line 9823 of file X86ISelLowering.cpp.
References canWidenShuffleElements(), llvm::DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), isShuffleMaskInputInPlace(), isSingleInputShuffleMask(), lowerV2X128VectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsShift(), lowerVectorShuffleByMerging128BitLanes(), llvm::X86ISD::PSHUFD, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v4i64, llvm::MVT::v8i32, and llvm::X86ISD::VPERMI.
Referenced by lower256BitVectorShuffle().
|
static |
Handle lowering of 64-lane 8-bit integer shuffles.
Definition at line 10456 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasBWI(), splitAndLowerVectorShuffle(), and llvm::MVT::v64i8.
Referenced by lower512BitVectorShuffle().
|
static |
Handle lowering of 8-lane 32-bit floating point shuffles.
Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2 isn't available.
Definition at line 9907 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i32, is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), isSingleInputShuffleMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleWithSHUFPS(), llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v8f32, llvm::MVT::v8i32, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, and llvm::X86ISD::VPERMV.
Referenced by lower256BitVectorShuffle().
|
static |
Handle lowering of 8-lane 64-bit floating point shuffles.
Definition at line 10338 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), isShuffleEquivalent(), splitAndLowerVectorShuffle(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::MVT::v8f64.
Referenced by lower512BitVectorShuffle().
|
static |
Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.
The lowering strategy is to try to form pairs of input lanes which are targeted at the same half of the final vector, and then use a dword shuffle to place them onto the right half, and finally unpack the paired lanes into their final position.
The exact breakdown of how to form these dword pairs and align them on the correct sides is really tricky. See the comments within the function for more of the details.
This code also handles repeated 128-bit lanes of v8i16 shuffles, but each lane must shuffle the exact same way. In fact, you must pass a v8 Mask to this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16 vector, form the analogous 128-bit 8-element Mask.
Definition at line 8218 of file X86ISelLowering.cpp.
References llvm::sys::path::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), llvm::MutableArrayRef< T >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::data(), llvm::ArrayRef< T >::empty(), llvm::sys::path::end(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::MutableArrayRef< T >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i16, llvm::MVT::i32, isNoopShuffleMask(), llvm_unreachable, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::MutableArrayRef< T >::slice(), std::swap(), and llvm::MVT::v8i16.
Referenced by lowerV16I16VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Generic lowering of 8-lane i16 shuffles.
This handles both single-input shuffles and combined shuffle/blends with two inputs. The single input shuffles are immediately delegated to a dedicated lowering routine.
The blends are lowered in one of three fundamental ways. If there are few enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle of the input is significantly cheaper when lowered as an interleaving of the two inputs, try to interleave them. Otherwise, blend the low and high halves of the inputs separately (making them have relatively few inputs) and then concatenate them.
Definition at line 8715 of file X86ISelLowering.cpp.
References llvm::MutableArrayRef< T >::begin(), llvm::DL, llvm::MutableArrayRef< T >::end(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), isShuffleEquivalent(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPSHUFB(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsUnpack(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::MVT::v8i16.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering of 8-lane 32-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v8i32 shuffling..
Definition at line 10006 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i32, is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), isSingleInputShuffleMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleByMerging128BitLanes(), llvm::X86ISD::PSHUFD, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v8i32, and llvm::X86ISD::VPERMV.
Referenced by lower256BitVectorShuffle().
|
static |
Handle lowering of 8-lane 64-bit integer shuffles.
Definition at line 10389 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), isShuffleEquivalent(), splitAndLowerVectorShuffle(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::MVT::v8i64.
Referenced by lower512BitVectorShuffle().
|
static |
Definition at line 15063 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemcpy(), llvm::SDValue::getOperand(), and llvm::X86Subtarget::is64Bit().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 12790 of file X86ISelLowering.cpp.
References llvm::SmallVectorTemplateCommon< T >::back(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::begin(), llvm::SmallVectorTemplateCommon< T >::begin(), llvm::DL, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE41(), I, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::insert(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::ISD::OR, llvm::MVT::Other, llvm::X86ISD::PTEST, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SmallVectorTemplateCommon< T >::size(), llvm::Slot, llvm::MVT::v2i64, and llvm::MVT::v4i64.
|
static |
LowerVectorBroadcast - Attempt to use the vbroadcast instruction to generate a splat value for the following cases:
Definition at line 5070 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::ISD::Constant, llvm::ISD::ConstantFP, llvm::HexagonISD::CP, Extract128BitVector(), F(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasInt256(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasVLX(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::ShuffleVectorSDNode::isSplat(), llvm::EVT::isVector(), llvm::Attribute::OptimizeForSize, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::VBROADCAST, and llvm::ISD::VECTOR_SHUFFLE.
|
static |
Definition at line 18253 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::DL, Extract128BitVector(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), LowerVectorCTPOPBitmath(), and LowerVectorCTPOPInRegLUT().
Referenced by LowerCTPOP().
|
static |
Definition at line 18182 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::APInt::getSplat(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i8, llvm::MVT::is128BitVector(), LowerHorizontalByteSum(), llvm::ISD::SRL, and llvm::ISD::SUB.
Referenced by LowerVectorCTPOP().
|
static |
Definition at line 18122 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::MVT::i8, LowerHorizontalByteSum(), llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), and llvm::ISD::SRL.
Referenced by LowerVectorCTPOP().
|
static |
Top-level lowering for x86 vector shuffles.
This handles decomposition, canonicalization, and lowering of all x86 vector shuffles. Most of the specific lowering strategies are encapsulated above in helper routines. The canonicalization attempts to widen shuffles to involve fewer lanes of wider elements, consolidate symmetric patterns s.t. only one of the two inputs needs to be tested, etc.
Definition at line 10527 of file X86ISelLowering.cpp.
References llvm::SmallBitVector::all(), canWidenShuffleElements(), computeZeroableShuffleElements(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getCommutedVectorShuffle(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::ShuffleVectorSDNode::getMask(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, lower128BitVectorShuffle(), lower256BitVectorShuffle(), lower512BitVectorShuffle(), llvm::ISD::UNDEF, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Try to emit a blend instruction for a shuffle using bit math.
This is used as a fallback approach when first class blend instructions are unavailable. Currently it is only suitable for integer vectors, but could be generalized for floating point vectors if desirable.
Definition at line 6430 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::ISD::BUILD_VECTOR, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::MVT::i64, llvm::MVT::isInteger(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I8VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Try to emit a bitmask instruction for a shuffle.
This handles cases where we can model a blend exactly as a bitmask due to one of the inputs being zeroable.
Definition at line 6836 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, computeZeroableShuffleElements(), llvm::X86ISD::FAND, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::MVT::isFloatingPoint(), llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV4I32VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Try to emit a blend instruction for a shuffle.
This doesn't do any checks for the availability of instructions for blending these values. It relies on the availability of the X86ISD::BLENDI pattern to be matched in the backend with the type given. What it does check for is that the shuffle mask is in fact a blend.
Definition at line 6462 of file X86ISelLowering.cpp.
References llvm::X86ISD::BLENDI, llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i8, is128BitLaneRepeatedShuffleMask(), llvm_unreachable, llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::MVT::SimpleTy, llvm::ArrayRef< T >::size(), llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8f32, llvm::MVT::v8i16, llvm::MVT::v8i32, and llvm::ISD::VSELECT.
Referenced by lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV2X128VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Try to lower as a blend of elements from two inputs followed by a single-input permutation.
This matches the pattern where we can blend elements from two inputs and then reduce the shuffle to a single-input permutation.
Definition at line 6591 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), and llvm::ArrayRef< T >::size().
Referenced by lowerV4F32VectorShuffle(), and lowerVectorShuffleAsDecomposedShuffleBlend().
|
static |
Try to lower broadcast of a single element.
For convenience, this code also bundles all of the subtarget feature set filtering. While a little annoying to re-dispatch on type here, there isn't a convenient way to factor it out.
Definition at line 7439 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ConstantSDNode::getZExtValue(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::isInteger(), isShuffleFoldableLoad(), llvm::ISD::SCALAR_TO_VECTOR, and llvm::X86ISD::VBROADCAST.
Referenced by lower512BitVectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Try to lower a vector shuffle as a byte rotation.
SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary byte-rotation of the concatenation of two vectors; pre-SSSE3 can use a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will try to generically lower a vector shuffle through such an pattern. It does not check for the profitability of lowering either as PALIGNR or PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form. This matches shuffle vectors that look like:
v8i16 [11, 12, 13, 14, 15, 0, 1, 2]
Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.
Definition at line 6675 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSSE3(), llvm::MipsISD::Hi, llvm::MVT::i8, isNoopShuffleMask(), llvm::MipsISD::Lo, llvm::ISD::OR, llvm::X86ISD::PALIGNR, llvm::ArrayRef< T >::size(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.
Referenced by lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Generic routine to decompose a shuffle and blend into indepndent blends and permutes.
This matches the extremely common pattern for handling combined shuffle+blend operations on newer X86 ISAs where we have very fast blend operations. It will try to pick the best arrangement of shuffles and blends.
Definition at line 6625 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), isNoopShuffleMask(), lowerVectorShuffleAsBlendAndPermute(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), lowerVectorShuffleAsLanePermuteAndBlend(), and lowerVectorShuffleAsSplitOrBlend().
|
static |
Try to lower insertion of a single element into a zero vector.
This is a common pattern that we have especially efficient patterns to lower across all subtarget feature sets.
Definition at line 7336 of file X86ISelLowering.cpp.
References llvm::ArrayRef< T >::begin(), computeZeroableShuffleElements(), llvm::ArrayRef< T >::end(), llvm::MVT::f32, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getScalarShiftAmountTy(), getScalarValueForVectorElement(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, llvm::ISD::SCALAR_TO_VECTOR, llvm::ArrayRef< T >::size(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VZEXT_MOVL, and llvm::ISD::ZERO_EXTEND.
Referenced by lower256BitVectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4I32VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Definition at line 7517 of file X86ISelLowering.cpp.
References computeZeroableShuffleElements(), llvm::DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::i8, llvm::X86ISD::INSERTPS, llvm::ArrayRef< T >::size(), and llvm::MVT::v4f32.
Referenced by lowerV4F32VectorShuffle().
|
static |
Lower a vector shuffle crossing multiple 128-bit lanes as a permutation and blend of those lanes.
This essentially blends the out-of-lane inputs to each lane into the lane from a permuted copy of the vector. This lowering strategy results in four instructions in the worst case for a single-input cross lane shuffle which is lower than any other fully general cross-lane shuffle strategy I'm aware of. Special cases for each particular shuffle pattern should be handled prior to trying this lowering.
Definition at line 9476 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::i8, isSingleInputShuffleMask(), lowerVectorShuffleAsDecomposedShuffleBlend(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ArrayRef< T >::size(), splitAndLowerVectorShuffle(), and llvm::X86ISD::VPERM2X128.
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), and lowerV8F32VectorShuffle().
|
static |
Helper to form a PSHUFB-based shuffle+blend.
Definition at line 8652 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, computeZeroableShuffleElements(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::i8, llvm::ISD::OR, llvm::X86ISD::PSHUFB, llvm::ArrayRef< T >::size(), llvm::MVT::v16i8, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16I8VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Try to lower a vector shuffle as a bit shift (shifts in zeros).
Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function matches elements from one of the input vectors shuffled to the left or right with zeroable elements 'shifted in'. It handles both the strictly bit-wise element shifts and the byte shift across an entire 128-bit double quad word lane.
PSHL : (little-endian) left bit shift. [ zz, 0, zz, 2 ] [ -1, 4, zz, -1 ] PSRL : (little-endian) right bit shift. [ 1, zz, 3, zz] [ -1, -1, 7, zz] PSLLDQ : (little-endian) left byte shift [ zz, 0, 1, 2, 3, 4, 5, 6] [ zz, zz, -1, -1, 2, 3, 4, -1] [ zz, zz, zz, zz, zz, zz, -1, 1] PSRLDQ : (little-endian) right byte shift [ 5, 6, 7, zz, zz, zz, zz, zz] [ -1, 5, 6, 7, zz, zz, zz, zz] [ 1, 2, -1, -1, -1, -1, zz, zz]
Definition at line 6897 of file X86ISelLowering.cpp.
References computeZeroableShuffleElements(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i8, isSequentialOrUndefInRange(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ArrayRef< T >::size(), llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VSRLI.
Referenced by lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Lower a vector shuffle as a zero or any extension.
Given a specific number of elements, element bit width, and extension stride, produce either a zero or any extension based on the available features of the subtarget.
Definition at line 7099 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::MipsISD::Ext, llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), llvm::MipsISD::Hi, llvm::MVT::i8, isUndefInRange(), llvm::MipsISD::Lo, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKL, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, and llvm::X86ISD::VZEXT.
Referenced by lowerVectorShuffleAsZeroOrAnyExtend().
|
static |
Either split a vector in halves or decompose the shuffles and the blend.
This is provided as a good fallback for many lowerings of non-single-input shuffles with more than one 128-bit lane. In those cases, we want to select between splitting the shuffle into 128-bit components and stitching those back together vs. extracting the single-input shuffles and blending those results.
Definition at line 9417 of file X86ISelLowering.cpp.
References llvm::SmallBitVector::count(), llvm::MVT::getSizeInBits(), isSingleInputShuffleMask(), lowerVectorShuffleAsDecomposedShuffleBlend(), llvm::SmallBitVector::resize(), llvm::ArrayRef< T >::size(), and splitAndLowerVectorShuffle().
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), and lowerV8F32VectorShuffle().
|
static |
Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.
This specifically targets cases where we end up with alternating between the two inputs, and so can permute them into something that feeds a single UNPCK instruction. Note that this routine only targets integer vectors because for floating point vectors we have a generalized SHUFPS lowering strategy that handles everything that doesn't exactly match an unpack, making this clever lowering unnecessary.
Definition at line 7598 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::assign(), llvm::ArrayRef< T >::begin(), llvm::ArrayRef< T >::end(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), isSingleInputShuffleMask(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by lowerV16I8VectorShuffle(), lowerV4I32VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Try to lower a vector shuffle as a zero extension on any microarch.
This routine will try to do everything in its power to cleverly lower a shuffle which happens to match the pattern of a zero extend. It doesn't check for the profitability of this lowering, it tries to aggressively match this pattern. It will use all of the micro-architectural details it can to emit an efficient lowering. It handles both blends with all-zero inputs to explicitly zero-extend and undef-lanes (sometimes undef due to masking out later).
The reason we have dedicated lowering for zext-style shuffles is that they are both incredibly common and often quite performance sensitive.
Definition at line 7202 of file X86ISelLowering.cpp.
References llvm::tgtok::Bits, computeZeroableShuffleElements(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), isSequentialOrUndefInRange(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), llvm::ArrayRef< T >::size(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, and llvm::X86ISD::VZEXT_MOVL.
Referenced by lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.
This will only succeed when the result of fixing the 128-bit lanes results in a single-input non-lane-crossing shuffle with a repeating shuffle mask in each 128-bit lanes. This handles many cases where we can quickly blend away the lane crosses early and then use simpler shuffles within each lane.
FIXME: It might be worthwhile at some point to support this without requiring the 128-bit lane-relative shuffles to be repeating, but currently in x86 only floating point has interesting non-repeating shuffles, and even those are still marginally more expensive.
Definition at line 9614 of file X86ISelLowering.cpp.
References llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MVT::i64, is128BitLaneCrossingShuffleMask(), llvm::MVT::isFloatingPoint(), isSingleInputShuffleMask(), llvm::SmallVectorImpl< T >::resize(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Definition at line 9700 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::i8, and llvm::X86ISD::SHUFP.
Referenced by lowerV4F64VectorShuffle().
|
static |
Lower a vector shuffle using the SHUFPS instruction.
This is a helper routine dedicated to lowering vector shuffles using SHUFPS. It makes no assumptions about whether this is the best lowering, it simply uses it.
Definition at line 7923 of file X86ISelLowering.cpp.
References llvm::ArrayRef< T >::begin(), llvm::ArrayRef< T >::end(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), llvm::X86ISD::SHUFP, std::swap(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV4F32VectorShuffle(), and lowerV8F32VectorShuffle().
|
static |
Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
Definition at line 6965 of file X86ISelLowering.cpp.
References llvm::SmallBitVector::all(), computeZeroableShuffleElements(), llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MipsISD::Hi, llvm::MVT::i8, llvm::X86ISD::INSERTQI, isSequentialOrUndefInRange(), isUndefInRange(), llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16I8VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Try to lower a VSELECT instruction to a vector shuffle.
Definition at line 10695 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::isBuildVectorOfConstantSDNodes(), and isZero().
|
static |
Definition at line 13532 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, ChangeVSETULTtoVSETULE(), llvm::X86ISD::CMPM, llvm::X86ISD::CMPP, EQ, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOperand(), llvm::APInt::getSignBit(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::AArch64CC::GT, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE42(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::MVT::isFloatingPoint(), llvm_unreachable, Lower256IntVSETCC(), LowerBoolVSETCC_AVX512(), LowerIntVSETCC_AVX512(), llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::X86ISD::SUBUS, std::swap(), translateX86FSETCC(), llvm::ISD::TRUNCATE, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::MVT::v2i64, llvm::MVT::v4i32, and llvm::ISD::XOR.
|
static |
Definition at line 17717 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADD, llvm::X86::COND_B, llvm::X86::COND_O, llvm::X86ISD::DEC, llvm::DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::MVT::i32, llvm::MVT::i8, llvm::X86ISD::INC, llvm::ConstantSDNode::isOne(), llvm_unreachable, llvm::ISD::MERGE_VALUES, llvm::ISD::SADDO, llvm::X86ISD::SETCC, llvm::X86ISD::SMUL, llvm::X86ISD::SMUL8, llvm::ISD::SMULO, llvm::ISD::SSUBO, llvm::X86ISD::SUB, llvm::ISD::UADDO, llvm::X86ISD::UMUL, llvm::X86ISD::UMUL8, llvm::ISD::UMULO, and llvm::ISD::USUBO.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 12410 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasFp256(), llvm::MVT::i1, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerAVXExtend(), and LowerZERO_EXTEND_AVX512().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 12375 of file X86ISelLowering.cpp.
References llvm::DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::APInt::getNullValue(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasBWI(), llvm::MVT::i1, llvm::MVT::is512BitVector(), One, llvm::MVT::v16i32, llvm::MVT::v8i64, llvm::ISD::VSELECT, llvm::X86ISD::VTRUNC, and llvm::X86ISD::VZEXT.
Referenced by LowerZERO_EXTEND().
Definition at line 25536 of file X86ISelLowering.cpp.
References llvm::StringRef::empty(), llvm::StringRef::find_first_not_of(), llvm::StringRef::startswith(), and llvm::StringRef::substr().
Referenced by llvm::X86TargetLowering::ExpandInlineAsm().
|
static |
MatchingStackOffset - Return true if the given stack call argument is already available in the same position (relatively) of the caller's incoming argument stack.
Definition at line 3387 of file X86ISelLowering.cpp.
References llvm::ISD::CopyFromReg, llvm::tgtok::Def, llvm::dyn_cast(), llvm::ISD::FrameIndex, llvm::ISD::ArgFlagsTy::getByValSize(), llvm::MachineOperand::getIndex(), llvm::FrameIndexSDNode::getIndex(), llvm::MachineFrameInfo::getObjectOffset(), llvm::MachineFrameInfo::getObjectSize(), llvm::SDValue::getOpcode(), llvm::MachineInstr::getOpcode(), llvm::SDValue::getOperand(), llvm::MachineInstr::getOperand(), getReg(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::MachineRegisterInfo::getVRegDef(), llvm::ISD::ArgFlagsTy::isByVal(), llvm::MachineOperand::isFI(), llvm::MachineFrameInfo::isFixedObjectIndex(), llvm::X86InstrInfo::isLoadFromStackSlot(), and llvm::TargetRegisterInfo::isVirtualRegister().
|
static |
Matches a VSELECT onto min/max or return 0 if the node doesn't match.
Definition at line 22012 of file X86ISelLowering.cpp.
References llvm::SDValue::getOperand(), llvm::EVT::getSimpleVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasVLX(), llvm::SelectionDAG::isEqualTo(), llvm::EVT::isVector(), llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::MVT::SimpleTy, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, and llvm::ISD::UMIN.
Referenced by PerformSELECTCombine().
|
static |
Definition at line 24985 of file X86ISelLowering.cpp.
References AND, llvm::X86::COND_B, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::X86ISD::SETCC_CARRY, and llvm::ISD::TRUNCATE.
Referenced by PerformSETCCCombine().
Definition at line 3662 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::hasOneUse(), llvm::ISD::isNormalStore(), and llvm::SDNode::use_begin().
Referenced by llvm::X86TargetLowering::IsDesirableToPromoteOp().
Definition at line 3658 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::hasOneUse(), and llvm::ISD::isNormalLoad().
Referenced by llvm::X86TargetLowering::IsDesirableToPromoteOp(), and PerformINSERTPSCombine().
|
static |
Definition at line 24910 of file X86ISelLowering.cpp.
References ADD, llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getStoreSize(), and llvm::MVT::getVectorElementType().
Referenced by PerformINSERTPSCombine().
|
static |
Definition at line 25206 of file X86ISelLowering.cpp.
References llvm::AArch64ISD::ADC, llvm::ARMISD::CMP, MSP430CC::COND_E, MSP430CC::COND_NE, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::isInteger(), llvm::X86::isZeroNode(), llvm::X86ISD::SBB, llvm::ISD::SETCC, SUB, and llvm::ISD::ZERO_EXTEND.
Referenced by PerformAddCombine(), and PerformSubCombine().
|
static |
Definition at line 25177 of file X86ISelLowering.cpp.
References AND, llvm::X86::COND_B, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86::isZeroNode(), llvm::X86ISD::SETCC_CARRY, and llvm::SDValue::use_empty().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
PerformADDCombine - Do target-specific dag combines on integer adds.
Definition at line 25244 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), isHorizontalBinOp(), and OptimizeConditionalInDecrement().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23630 of file X86ISelLowering.cpp.
References llvm::X86ISD::ANDNP, llvm::X86ISD::BEXTR, CanFoldXORWithAllOnes(), CMPEQCombine(), llvm::countPopulation(), llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::X86Subtarget::hasBMI(), llvm::X86Subtarget::hasTBM(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isMask_64(), llvm::ISD::SRA, llvm::ISD::SRL, VectorZextCombine(), and XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Detect bitcasts between i32 to x86mmx low word.
Since MMX types are special and don't usually play with other vector types, it's better to handle them early to be sure we emit efficient code by avoiding store-load conversions.
Definition at line 21844 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::ConstantSDNode::getZExtValue(), and llvm::X86ISD::MMX_MOVW2D.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24956 of file X86ISelLowering.cpp.
References llvm::X86ISD::BLENDI, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::APInt::getZExtValue(), and isShuffleFoldableLoad().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 25042 of file X86ISelLowering.cpp.
References llvm::ISD::BRCOND, checkBoolTestSetCCCombine(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), and llvm::SDNode::getVTList().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24580 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and llvm::Log2_32().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].
Definition at line 22867 of file X86ISelLowering.cpp.
References ADD, llvm::X86ISD::BSF, llvm::X86ISD::BSR, checkBoolTestAndOrSetCCCombine(), checkBoolTestSetCCCombine(), llvm::ARMISD::CMOV, llvm::ARMISD::CMP, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), MSP430CC::COND_E, MSP430CC::COND_NE, llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), GetOppositeBranchCondition(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDNode::getVTList(), hasFPCMov(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::SelectionDAG::isKnownNeverZero(), llvm::ISD::MUL, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SETCC, llvm::ISD::SHL, SUB, std::swap(), llvm::SDValue::use_empty(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
PerformEXTRACT_VECTOR_ELTCombine - Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.
For i686, the best sequence is apparently storing the value and loading scalars back, while for x64 we should use 64-bit extracts and shifts.
Definition at line 21864 of file X86ISelLowering.cpp.
References ADD, llvm::SmallVectorTemplateCommon< T >::begin(), llvm::ISD::BITCAST, llvm::SelectionDAG::CreateStackTemporary(), llvm::dyn_cast(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SDValue::getResNo(), llvm::TargetLoweringBase::getShiftAmountTy(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::TargetLoweringBase::getVectorIdxTy(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::TargetLoweringBase::isOperationLegal(), llvm::X86ISD::MMX_MOVD2W, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), XFormVExtractWithShuffleIntoLoad(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on floating point adds.
Definition at line 24484 of file X86ISelLowering.cpp.
References llvm::X86ISD::FHADD, llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasFp256(), llvm::X86Subtarget::hasSSE3(), and isHorizontalBinOp().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on X86ISD::FAND nodes.
Definition at line 24551 of file X86ISelLowering.cpp.
References llvm::SDNode::getOperand().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on X86ISD::FANDN nodes.
Definition at line 24566 of file X86ISelLowering.cpp.
References llvm::SDNode::getOperand().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24747 of file X86ISelLowering.cpp.
References llvm::ARM_PROC::A, llvm::X86ISD::FMADD, llvm::X86ISD::FMSUB, llvm::ISD::FNEG, llvm::X86ISD::FNMADD, llvm::X86ISD::FNMSUB, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasFMA(), llvm::X86Subtarget::hasFMA4(), and llvm::TargetLoweringBase::isTypeLegal().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
Definition at line 24530 of file X86ISelLowering.cpp.
References llvm::AArch64ISD::FMAX, llvm::X86ISD::FMAXC, llvm::AArch64ISD::FMIN, llvm::X86ISD::FMINC, llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTarget(), llvm::SDNode::getValueType(), llvm_unreachable, llvm::TargetMachine::Options, and llvm::TargetOptions::UnsafeFPMath.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
Definition at line 24514 of file X86ISelLowering.cpp.
References llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SDNode::getOpcode(), and llvm::SDNode::getOperand().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on floating point subs.
Definition at line 24499 of file X86ISelLowering.cpp.
References llvm::X86ISD::FHSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasFp256(), llvm::X86Subtarget::hasSSE3(), and isHorizontalBinOp().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24928 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::X86ISD::INSERTPS, MayFoldLoad(), NarrowVectorLoadToElement(), and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23854 of file X86ISelLowering.cpp.
References ADD, llvm::ARMISD::CMOV, MSP430CC::COND_GE, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isInteger(), llvm::ISD::SRA, SUB, and XOR.
Referenced by PerformXorCombine().
|
static |
Definition at line 23076 of file X86ISelLowering.cpp.
References llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::ConstantSDNode::isAllOnesValue(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ConstantSDNode::isNullValue(), llvm::EVT::isVector(), and llvm::ISD::SRA.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24849 of file X86ISelLowering.cpp.
References ADD, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCSwappedOperands(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isSignedIntSetCC(), llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, SUB, and std::swap().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
PerformLOADCombine - Do target-specific dag combines on LOAD nodes.
Definition at line 23902 of file X86ISelLowering.cpp.
References ADD, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::LoadSDNode::getExtensionType(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), Insert128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::MemSDNode::isInvariant(), llvm::MemSDNode::isNonTemporal(), llvm::X86Subtarget::isUnalignedMem32Slow(), llvm::MemSDNode::isVolatile(), fuzzer::min(), N, llvm::ISD::NON_EXTLOAD, Other, and llvm::ISD::TokenFactor.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
PerformMLOADCombine - Resolve extending loads.
Definition at line 23951 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, llvm::MaskedLoadStoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MaskedLoadSDNode::getSrc0(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), N, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::UNDEF, and llvm::X86ISD::VSEXT.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
PerformMSTORECombine - Resolve truncating stores.
Definition at line 24032 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::MaskedLoadStoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MaskedStoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::isPowerOf2_32(), llvm::MaskedStoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTypeLegal(), and N.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
PerformMulCombine - Optimize a single multiply with constant into two in order to implement it with two cheaper instructions, e.g.
LEA + SHL, LEA + LEA.
Definition at line 23174 of file X86ISelLowering.cpp.
References ADD, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::SDNode::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm::isPowerOf2_64(), llvm::Log2_64(), llvm::X86ISD::MUL_IMM, llvm::ISD::SHL, std::swap(), and llvm::SDNode::use_begin().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23693 of file X86ISelLowering.cpp.
References AND, llvm::X86ISD::ANDNP, llvm::ISD::BITCAST, llvm::tgtok::Bits, CMPEQCombine(), llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::ConstantSDNode::getSExtValue(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasInt256(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllZeros(), llvm::X86Subtarget::isSHLDSlow(), llvm::X86ISD::PSIGN, llvm::ISD::SHL, llvm::X86ISD::SHLD, llvm::X86ISD::SHRD, llvm::ISD::SRA, llvm::ISD::SRL, SUB, std::swap(), llvm::ISD::TRUNCATE, llvm::ISD::VSELECT, llvm::X86ISD::VSRAI, X, and Y.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
PerformSELECTCombine - Do target-specific dag combines on SELECT and VSELECT nodes.
Definition at line 22146 of file X86ISelLowering.cpp.
References ADD, llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), AND, llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), llvm::ISD::CONCAT_VECTORS, Extract128BitVector(), llvm::AArch64ISD::FMAX, llvm::AArch64ISD::FMIN, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasVLX(), I, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SelectionDAG::isEqualTo(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::SelectionDAG::isKnownNeverZero(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), matchIntegerMINMAX(), llvm::ISD::MUL, llvm::TargetMachine::Options, OR, Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::MipsISD::Ret, llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::X86ISD::SHRUNKBLEND, llvm::ISD::SIGN_EXTEND, llvm::TargetLowering::SimplifyDemandedBits(), SUB, llvm::X86ISD::SUBUS, std::swap(), transformVSELECTtoBlendVECTOR_SHUFFLE(), llvm::TargetOptions::UnsafeFPMath, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::ISD::VSELECT, XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 25001 of file X86ISelLowering.cpp.
References checkBoolTestSetCCCombine(), llvm::X86::COND_A, llvm::X86::COND_B, llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SDNode::getVTList(), MaterializeSETB(), llvm::ISD::SETCC, and SUB.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24647 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getResNo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSignExtendVectorInReg(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasFp256(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SDIVREM, llvm::X86ISD::SDIVREM8_SEXT_HREG, llvm::ISD::SELECT, llvm::ISD::SIGN_EXTEND, and WidenMaskArithmetic().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
PerformShiftCombine - Combine shifts.
Definition at line 23309 of file X86ISelLowering.cpp.
References llvm::SDNode::getOpcode(), performShiftToAllZeros(), PerformSHLCombine(), llvm::ISD::SHL, and llvm::ISD::SRA.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Returns a vector of 0s if the node in input is a vector logical shift by a constant amount which is known to be bigger than or equal to the vector element size in bits.
Definition at line 23281 of file X86ISelLowering.cpp.
References llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), getZeroVector(), llvm::X86Subtarget::hasInt256(), llvm::APInt::trunc(), and llvm::APInt::uge().
Referenced by PerformShiftCombine().
|
static |
Definition at line 23234 of file X86ISelLowering.cpp.
References ADD, AND, llvm::ISD::ANY_EXTEND, llvm::ISD::Constant, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::isVector(), llvm::X86ISD::SETCC_CARRY, llvm::APInt::shl(), and llvm::ISD::ZERO_EXTEND.
Referenced by PerformShiftCombine().
|
static |
PerformShuffleCombine - Performs several different shuffle combines.
Definition at line 21636 of file X86ISelLowering.cpp.
References ADD, llvm::ISD::BITCAST, combineShuffleToAddSub(), combineX86ShufflesRecursively(), EltsFromConsecutiveLoads(), llvm::ISD::FADD, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::ShuffleVectorSDNode::getMask(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), getShuffleScalarElt(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasFp256(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE3(), llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isOperationLegal(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::AArch64DB::LD, llvm::ISD::MUL, N, PerformShuffleCombine256(), PerformTargetShuffleCombine(), llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), SUB, llvm::ISD::UNDEF, and llvm::ISD::VECTOR_SHUFFLE.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
PerformShuffleCombine256 - Performs shuffle combines for 256-bit vectors.
Definition at line 20821 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, Extract128BitVector(), llvm::SelectionDAG::getBitcast(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), getZeroVector(), Insert128BitVector(), llvm::ISD::isBuildVectorAllZeros(), isShuffleHigh128VectorInsertLow(), isShuffleLow128VectorInsertHigh(), isUndefOrEqual(), N, llvm::MVT::Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::TokenFactor, llvm::ISD::UNDEF, llvm::SelectionDAG::UpdateNodeOperands(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v4i64, and llvm::X86ISD::VZEXT_LOAD.
Referenced by PerformShuffleCombine().
|
static |
Definition at line 24612 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasInt256(), llvm::ISD::isNormalLoad(), llvm::EVT::isVector(), llvm::ISD::LOAD, llvm::ISD::SIGN_EXTEND, and llvm::ISD::SIGN_EXTEND_INREG.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 25131 of file X86ISelLowering.cpp.
References llvm::X86TargetLowering::BuildFILD(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::X86Subtarget::getTargetLowering(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::is64Bit(), llvm::ISD::isNON_EXTLoad(), llvm::EVT::isVector(), llvm::MemSDNode::isVolatile(), llvm::ISD::LOAD, P, performVectorCompareAndMaskUnaryOpCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::SINT_TO_FP.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
PerformSTORECombine - Do target-specific dag combines on STORE nodes.
Definition at line 24111 of file X86ISelLowering.cpp.
References ADD, Extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), llvm::MachinePointerInfo::getWithOffset(), llvm::Function::hasFnAttribute(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::is64Bit(), llvm::MemSDNode::isInvariant(), llvm::MemSDNode::isNonTemporal(), llvm::ISD::isNormalLoad(), llvm::isPowerOf2_32(), llvm::StoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86Subtarget::isUnalignedMem32Slow(), isVolatile(), llvm::MemSDNode::isVolatile(), fuzzer::min(), llvm::MinAlign(), N, Other, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ISD::TokenFactor, and llvm::X86Subtarget::useSoftFloat().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 25259 of file X86ISelLowering.cpp.
References ADD, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, isHorizontalBinOp(), OptimizeConditionalInDecrement(), and XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Try to combine x86 target specific shuffles.
Definition at line 21480 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::BITCAST, combineRedundantDWordShuffle(), combineRedundantHalfShuffle(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPSHUFShuffleMask(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), isNoopShuffleMask(), llvm::makeArrayRef(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by PerformShuffleCombine().
|
static |
Definition at line 25106 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::TargetLoweringBase::isOperationLegal(), llvm::EVT::isVector(), P, llvm::ISD::SINT_TO_FP, llvm::ISD::UINT_TO_FP, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 25060 of file X86ISelLowering.cpp.
References AND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::isVector(), and llvm::ISD::SETCC.
Referenced by PerformSINT_TO_FPCombine().
|
static |
Definition at line 24599 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), and llvm::X86ISD::VZEXT_LOAD.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
performVZEXTCombine - Performs build vector combines
Definition at line 25293 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::ISD::SCALAR_TO_VECTOR, and llvm::X86ISD::VZEXT.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23888 of file X86ISelLowering.cpp.
References llvm::X86Subtarget::hasCMov(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and performIntegerAbsCombine().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24788 of file X86ISelLowering.cpp.
References AND, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getResNo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::ConstantSDNode::getZExtValue(), llvm::SDValue::hasOneUse(), llvm::EVT::is256BitVector(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::X86ISD::SETCC_CARRY, llvm::ISD::TRUNCATE, llvm::ISD::UDIVREM, llvm::X86ISD::UDIVREM8_ZEXT_HREG, and WidenMaskArithmetic().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
When the 32-bit MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP.
Here's the math: RegNodeBase = EntryEBP - RegNodeSize ParentFP = RegNodeBase - RegNodeFrameOffset Subtracting RegNodeSize takes us to the offset of the registration node, and subtracting the offset (negative on x86) takes us back to the parent FP.
Definition at line 15294 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::MachineModuleInfo::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMCSymbol(), llvm::MachineFunction::getMMI(), llvm::Value::getName(), llvm::SelectionDAG::getNode(), llvm::MCContext::getOrCreateParentFrameOffsetSymbol(), llvm::TargetLoweringBase::getPointerTy(), llvm::GlobalValue::getRealLinkageName(), getSEHRegistrationNodeSize(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasPersonalityFn(), llvm::ISD::LOCAL_RECOVER, and llvm::ISD::SUB.
Referenced by LowerINTRINSIC_WO_CHAIN(), and LowerSEHRESTOREFRAME().
|
static |
Generic routine to split vector shuffle into half-sized shuffles.
This routine just extracts two subvectors, shuffles them independently, and then concatenates them back together. This should work effectively with all AVX vector shuffle types.
Definition at line 9286 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MipsISD::Hi, llvm::MipsISD::Lo, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ArrayRef< T >::size(), and llvm::ArrayRef< T >::slice().
Referenced by lower256BitVectorShuffle(), lower512BitVectorShuffle(), lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV32I16VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I64VectorShuffle(), lowerVectorShuffleAsLanePermuteAndBlend(), and lowerVectorShuffleAsSplitOrBlend().
| STATISTIC | ( | NumTailCalls | , |
| "Number of tail calls" | |||
| ) |
|
static |
Definition at line 17030 of file X86ISelLowering.cpp.
References SupportedVectorShiftWithImm().
Referenced by LowerScalarVariableShift().
|
static |
Definition at line 17010 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasVLX(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::SRA, llvm::MVT::v2i64, and llvm::MVT::v4i64.
Referenced by LowerScalarImmediateShift(), and SupportedVectorShiftWithBaseAmnt().
|
static |
Definition at line 17037 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasVLX(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::SRA, llvm::MVT::v2i64, and llvm::MVT::v4i64.
Referenced by LowerShift().
|
static |
Definition at line 22101 of file X86ISelLowering.cpp.
References BUILD_VECTORtoBlendMask(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::TargetLoweringBase::isShuffleMaskLegal(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, and llvm::ISD::UNDEF.
Referenced by PerformSELECTCombine().
|
static |
TranslateX86CC - do a one to one translation of a ISD::CondCode to the X86 specific condition code, returning the condition code and the LHS/RHS of the comparison to make.
Definition at line 3816 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_INVALID, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_NS, llvm::X86::COND_P, llvm::X86::COND_S, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::ISD::getSetCCSwappedOperands(), llvm::SDValue::getValueType(), llvm::ISD::isNON_EXTLoad(), llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, and std::swap().
Referenced by LowerINTRINSIC_WO_CHAIN().
|
static |
Definition at line 13346 of file X86ISelLowering.cpp.
References llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, and std::swap().
Referenced by LowerVSETCC().
|
static |
Definition at line 23540 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::BuildVectorSDNode::isConstantSplat(), llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::ISD::UNDEF, and llvm::ISD::VECTOR_SHUFFLE.
Referenced by PerformAndCombine().
|
static |
Definition at line 23461 of file X86ISelLowering.cpp.
References AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::TargetLoweringBase::isOperationLegalOrPromote(), llvm_unreachable, OR, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::TRUNCATE, XOR, llvm::ISD::ZERO_EXTEND, and llvm::APInt::zext().
Referenced by PerformSExtCombine(), and PerformZExtCombine().
|
static |
XFormVExtractWithShuffleIntoLoad - Check if a vector extract from a target specific shuffle of a load can be folded into a single element load.
Similar handling for VECTOR_SHUFFLE is performed by DAGCombiner, but shuffles have been custom lowered so we need to handle those here.
Definition at line 21747 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::DataLayout::getABITypeAlignment(), llvm::MemSDNode::getAlignment(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::EVT::getTypeForEVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isNormalLoad(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), isTargetShuffle(), llvm::EVT::isVector(), llvm::MemSDNode::isVolatile(), and llvm::ISD::LOAD.
Referenced by PerformEXTRACT_VECTOR_ELTCombine().
1.8.6