|
LLVM
4.0.0
|
#include "X86ISelLowering.h"#include "Utils/X86ShuffleDecode.h"#include "X86CallingConv.h"#include "X86FrameLowering.h"#include "X86InstrBuilder.h"#include "X86IntrinsicsInfo.h"#include "X86MachineFunctionInfo.h"#include "X86ShuffleDecodeConstantPool.h"#include "X86TargetMachine.h"#include "X86TargetObjectFile.h"#include "llvm/ADT/SmallBitVector.h"#include "llvm/ADT/SmallSet.h"#include "llvm/ADT/Statistic.h"#include "llvm/ADT/StringExtras.h"#include "llvm/ADT/StringSwitch.h"#include "llvm/Analysis/EHPersonalities.h"#include "llvm/CodeGen/IntrinsicLowering.h"#include "llvm/CodeGen/MachineFrameInfo.h"#include "llvm/CodeGen/MachineFunction.h"#include "llvm/CodeGen/MachineInstrBuilder.h"#include "llvm/CodeGen/MachineJumpTableInfo.h"#include "llvm/CodeGen/MachineModuleInfo.h"#include "llvm/CodeGen/MachineRegisterInfo.h"#include "llvm/CodeGen/WinEHFuncInfo.h"#include "llvm/IR/CallSite.h"#include "llvm/IR/CallingConv.h"#include "llvm/IR/Constants.h"#include "llvm/IR/DerivedTypes.h"#include "llvm/IR/Function.h"#include "llvm/IR/GlobalAlias.h"#include "llvm/IR/GlobalVariable.h"#include "llvm/IR/Instructions.h"#include "llvm/IR/Intrinsics.h"#include "llvm/MC/MCAsmInfo.h"#include "llvm/MC/MCContext.h"#include "llvm/MC/MCExpr.h"#include "llvm/MC/MCSymbol.h"#include "llvm/Support/CommandLine.h"#include "llvm/Support/Debug.h"#include "llvm/Support/ErrorHandling.h"#include "llvm/Support/MathExtras.h"#include "llvm/Target/TargetOptions.h"#include <algorithm>#include <bitset>#include <cctype>#include <numeric>#include "X86GenCallingConv.inc"Go to the source code of this file.
Macros | |
| #define | DEBUG_TYPE "x86-isel" |
Enumerations | |
| enum | StructReturnType { NotStructReturn, RegStructReturn, StackStructReturn } |
| CallIsStructReturn - Determines whether a call uses struct return semantics. More... | |
| enum | ShrinkMode { MULS8, MULU8, MULS16, MULU16 } |
| Different mul shrinking modes. More... | |
Functions | |
| STATISTIC (NumTailCalls,"Number of tail calls") | |
| static void | getMaxByValAlign (Type *Ty, unsigned &MaxAlign) |
| Helper for getByValTypeAlignment to determine the desired ByVal argument alignment. More... | |
| static SDValue | lowerMasksToReg (const SDValue &ValArg, const EVT &ValLoc, const SDLoc &Dl, SelectionDAG &DAG) |
| Lowers masks values (v*i1) to the local register values. More... | |
| static void | Passv64i1ArgInRegs (const SDLoc &Dl, SelectionDAG &DAG, SDValue Chain, SDValue &Arg, SmallVector< std::pair< unsigned, SDValue >, 8 > &RegsToPass, CCValAssign &VA, CCValAssign &NextVA, const X86Subtarget &Subtarget) |
| Breaks v64i1 value into two registers and adds the new node to the DAG. More... | |
| static SDValue | getv64i1Argument (CCValAssign &VA, CCValAssign &NextVA, SDValue &Root, SelectionDAG &DAG, const SDLoc &Dl, const X86Subtarget &Subtarget, SDValue *InFlag=nullptr) |
| Reads two 32 bit registers and creates a 64 bit mask value. More... | |
| static SDValue | lowerRegToMasks (const SDValue &ValArg, const EVT &ValVT, const EVT &ValLoc, const SDLoc &Dl, SelectionDAG &DAG) |
| The function will lower a register of various sizes (8/16/32/64) to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1) More... | |
| static StructReturnType | callIsStructReturn (const SmallVectorImpl< ISD::OutputArg > &Outs, bool IsMCU) |
| static StructReturnType | argsAreStructReturn (const SmallVectorImpl< ISD::InputArg > &Ins, bool IsMCU) |
| Determines whether a function uses struct return semantics. More... | |
| static SDValue | CreateCopyOfByValArgument (SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl) |
| Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute. More... | |
| static bool | canGuaranteeTCO (CallingConv::ID CC) |
| Return true if the calling convention is one that we can guarantee TCO for. More... | |
| static bool | mayTailCallThisCC (CallingConv::ID CC) |
| Return true if we might ever do TCO for calls with this calling convention. More... | |
| static bool | shouldGuaranteeTCO (CallingConv::ID CC, bool GuaranteedTailCallOpt) |
| Return true if the function is being made into a tailcall target by changing its ABI. More... | |
| static ArrayRef< MCPhysReg > | get64BitArgumentGPRs (CallingConv::ID CallConv, const X86Subtarget &Subtarget) |
| static ArrayRef< MCPhysReg > | get64BitArgumentXMMs (MachineFunction &MF, CallingConv::ID CallConv, const X86Subtarget &Subtarget) |
| static bool | isSortedByValueNo (const SmallVectorImpl< CCValAssign > &ArgLocs) |
| static SDValue | EmitTailCallStoreRetAddr (SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue RetAddrFrIdx, EVT PtrVT, unsigned SlotSize, int FPDiff, const SDLoc &dl) |
| Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0). More... | |
| static SDValue | getMOVL (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2) |
| Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width. More... | |
| static bool | MatchingStackOffset (SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const X86InstrInfo *TII, const CCValAssign &VA) |
| Return true if the given stack call argument is already available in the same position (relatively) of the caller's incoming argument stack. More... | |
| static bool | MayFoldLoad (SDValue Op) |
| static bool | MayFoldIntoStore (SDValue Op) |
| static bool | MayFoldIntoZeroExtend (SDValue Op) |
| static bool | isTargetShuffle (unsigned Opcode) |
| static bool | isTargetShuffleVariableMask (unsigned Opcode) |
| static bool | isX86CCUnsigned (unsigned X86CC) |
| Return true if the condition is an unsigned comparison operation. More... | |
| static X86::CondCode | TranslateIntegerX86CC (ISD::CondCode SetCCOpcode) |
| static X86::CondCode | TranslateX86CC (ISD::CondCode SetCCOpcode, const SDLoc &DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) |
| Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make. More... | |
| static bool | hasFPCMov (unsigned X86CC) |
| Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. More... | |
| static bool | isUndefOrEqual (int Val, int CmpVal) |
| Val is the undef sentinel value or equal to the specified value. More... | |
| static bool | isUndefOrZero (int Val) |
| Val is either the undef or zero sentinel value. More... | |
| static bool | isUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size) |
| Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value. More... | |
| static bool | isUndefOrInRange (int Val, int Low, int Hi) |
| Return true if Val is undef or if its value falls within the specified range (L, H]. More... | |
| static bool | isUndefOrInRange (ArrayRef< int > Mask, int Low, int Hi) |
| Return true if every element in Mask is undef or if its value falls within the specified range (L, H]. More... | |
| static bool | isUndefOrZeroOrInRange (int Val, int Low, int Hi) |
| Return true if Val is undef, zero or if its value falls within the specified range (L, H]. More... | |
| static bool | isUndefOrZeroOrInRange (ArrayRef< int > Mask, int Low, int Hi) |
| Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H]. More... | |
| static bool | isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low) |
| Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size]. More... | |
| static bool | isSequentialOrUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low) |
| Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero. More... | |
| static bool | isUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size) |
| Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero. More... | |
| static bool | canWidenShuffleElements (ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask) |
| Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled. More... | |
| static void | scaleShuffleMask (int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask) |
| Helper function to scale a shuffle or target shuffle mask, replacing each mask index with the scaled sequential indices for an equivalent narrowed mask. More... | |
| static bool | isVEXTRACTIndex (SDNode *N, unsigned vecWidth) |
| Return true if the specified EXTRACT_SUBVECTOR operand specifies a vector extract that is suitable for instruction that extract 128 or 256 bit vectors. More... | |
| static bool | isVINSERTIndex (SDNode *N, unsigned vecWidth) |
| Return true if the specified INSERT_SUBVECTOR operand specifies a subvector insert that is suitable for input to insertion of 128 or 256-bit subvectors. More... | |
| static unsigned | getExtractVEXTRACTImmediate (SDNode *N, unsigned vecWidth) |
| static unsigned | getInsertVINSERTImmediate (SDNode *N, unsigned vecWidth) |
| static SDValue | getConstVector (ArrayRef< int > Values, MVT VT, SelectionDAG &DAG, const SDLoc &dl, bool IsMask=false) |
| static SDValue | getConstVector (ArrayRef< APInt > Bits, SmallBitVector &Undefs, MVT VT, SelectionDAG &DAG, const SDLoc &dl) |
| static SDValue | getZeroVector (MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl) |
| Returns a vector of specified type with all zero elements. More... | |
| static SDValue | extractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth) |
| static SDValue | extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl) |
| Generate a DAG to grab 128-bits from a vector > 128 bits. More... | |
| static SDValue | extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl) |
| Generate a DAG to grab 256-bits from a 512-bit vector. More... | |
| static SDValue | insertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth) |
| static SDValue | insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl) |
| Generate a DAG to put 128-bits into a vector > 128 bits. More... | |
| static SDValue | insert256BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl) |
| static SDValue | insert1BitVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Insert i1-subvector to i1-vector. More... | |
| static SDValue | concat128BitVectors (SDValue V1, SDValue V2, EVT VT, unsigned NumElems, SelectionDAG &DAG, const SDLoc &dl) |
| Concat two 128-bit vectors into a 256 bit vector using VINSERTF128 instructions. More... | |
| static SDValue | concat256BitVectors (SDValue V1, SDValue V2, EVT VT, unsigned NumElems, SelectionDAG &DAG, const SDLoc &dl) |
| static SDValue | getOnesVector (EVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl) |
| Returns a vector of specified type with all bits set. More... | |
| static void | createUnpackShuffleMask (MVT VT, SmallVectorImpl< int > &Mask, bool Lo, bool Unary) |
| Generate unpacklo/unpackhi shuffle mask. More... | |
| static SDValue | getUnpackl (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2) |
| Returns a vector_shuffle node for an unpackl operation. More... | |
| static SDValue | getUnpackh (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2) |
| Returns a vector_shuffle node for an unpackh operation. More... | |
| static SDValue | getShuffleVectorZeroOrUndef (SDValue V2, int Idx, bool IsZero, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Return a vector_shuffle of the specified vector of zero or undef vector. More... | |
| static SDValue | peekThroughBitcasts (SDValue V) |
| static SDValue | peekThroughOneUseBitcasts (SDValue V) |
| static const Constant * | getTargetConstantFromNode (SDValue Op) |
| static bool | getTargetConstantBitsFromNode (SDValue Op, unsigned EltSizeInBits, SmallBitVector &UndefElts, SmallVectorImpl< APInt > &EltBits) |
| static bool | getTargetShuffleMaskIndices (SDValue MaskNode, unsigned MaskEltSizeInBits, SmallVectorImpl< uint64_t > &RawMask) |
| static bool | getTargetShuffleMask (SDNode *N, MVT VT, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask, bool &IsUnary) |
| Calculates the shuffle mask corresponding to the target-specific opcode. More... | |
| static bool | setTargetShuffleZeroElements (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops) |
| Check a target shuffle mask's inputs to see if we can set any values to SM_SentinelZero - this is for elements that are known to be zero (not just zeroable) from their inputs. More... | |
| static bool | getFauxShuffleMask (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops) |
| static bool | resolveTargetShuffleInputs (SDValue Op, SDValue &Op0, SDValue &Op1, SmallVectorImpl< int > &Mask) |
| Calls setTargetShuffleZeroElements to resolve a target shuffle mask's inputs and set the SM_SentinelUndef and SM_SentinelZero values. More... | |
| static SDValue | getShuffleScalarElt (SDNode *N, unsigned Index, SelectionDAG &DAG, unsigned Depth) |
| Returns the scalar element that will make up the ith element of the result of the vector shuffle. More... | |
| static SDValue | LowerBuildVectorv16i8 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget, const TargetLowering &TLI) |
| Custom lower build_vector of v16i8. More... | |
| static SDValue | LowerBuildVectorv8i16 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget, const TargetLowering &TLI) |
| Custom lower build_vector of v8i16. More... | |
| static SDValue | LowerBuildVectorv4x32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget, const TargetLowering &TLI) |
| Custom lower build_vector of v4i32 or v4f32. More... | |
| static SDValue | getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &dl) |
| Return a vector logical shift node. More... | |
| static SDValue | LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, const SDLoc &dl, SelectionDAG &DAG) |
| static SDValue | EltsFromConsecutiveLoads (EVT VT, ArrayRef< SDValue > Elts, SDLoc &DL, SelectionDAG &DAG, bool isAfterLegalize) |
| Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'. More... | |
| static Constant * | getConstantVector (MVT VT, APInt SplatValue, unsigned SplatBitSize, LLVMContext &C) |
| static bool | isUseOfShuffle (SDNode *N) |
| static SDValue | LowerVectorBroadcast (BuildVectorSDNode *BVOp, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Attempt to use the vbroadcast instruction to generate a splat value for the following cases: More... | |
| static int | getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx) |
| For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index. More... | |
| static SDValue | buildFromShuffleMostly (SDValue Op, SelectionDAG &DAG) |
| static SDValue | ConvertI1VectorToInteger (SDValue Op, SelectionDAG &DAG) |
| static bool | isHorizontalBinOp (const BuildVectorSDNode *N, unsigned Opcode, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1) |
Return true if N implements a horizontal binop and return the operands for the horizontal binop into V0 and V1. More... | |
| static SDValue | ExpandHorizontalBinOp (const SDValue &V0, const SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI) |
| Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector. More... | |
| static bool | isAddSub (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1) |
Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB operation. More... | |
| static bool | isFMAddSub (const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2) |
Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB(Opnd0, Opnd1) into FMADDSUB(x, y, Opnd1). More... | |
| static SDValue | lowerToAddSubOrFMAddSub (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB node. More... | |
| static SDValue | LowerToHorizontalOp (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower BUILD_VECTOR to a horizontal add/sub operation if possible. More... | |
| static SDValue | lowerBuildVectorToBitOp (BuildVectorSDNode *Op, SelectionDAG &DAG) |
| If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors. More... | |
| static SDValue | materializeVectorConstant (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Create a vector constant without a load. More... | |
| static SDValue | LowerAVXCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerCONCAT_VECTORSvXi1 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCONCAT_VECTORS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | isNoopShuffleMask (ArrayRef< int > Mask) |
| Tiny helper function to identify a no-op mask. More... | |
| static bool | is128BitLaneCrossingShuffleMask (MVT VT, ArrayRef< int > Mask) |
| Test whether there are elements crossing 128-bit lanes in this shuffle mask. More... | |
| static bool | isRepeatedShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
| Test whether a shuffle mask is equivalent within each sub-lane. More... | |
| static bool | is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
| Test whether a shuffle mask is equivalent within each 128-bit lane. More... | |
| static bool | is256BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
| Test whether a shuffle mask is equivalent within each 256-bit lane. More... | |
| static bool | isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
| Test whether a target shuffle mask is equivalent within each sub-lane. More... | |
| static bool | isShuffleEquivalent (SDValue V1, SDValue V2, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask) |
| Checks whether a shuffle mask is equivalent to an explicit list of arguments. More... | |
| static bool | isTargetShuffleEquivalent (ArrayRef< int > Mask, ArrayRef< int > ExpectedMask) |
| Checks whether a target shuffle mask is equivalent to an explicit pattern. More... | |
| static unsigned | getV4X86ShuffleImm (ArrayRef< int > Mask) |
| Get a 4-lane 8-bit shuffle immediate for a mask. More... | |
| static SDValue | getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, SDLoc DL, SelectionDAG &DAG) |
| static SmallBitVector | computeZeroableShuffleElements (ArrayRef< int > Mask, SDValue V1, SDValue V2) |
| Compute whether each element of a shuffle is zeroable. More... | |
| static bool | isNonZeroElementsInOrder (const SmallBitVector Zeroable, ArrayRef< int > Mask, const EVT &VectorType, bool &IsZeroSideLeft) |
| static SDValue | lowerVectorShuffleWithPSHUFB (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower a shuffle with a single PSHUFB of V1 or V2. More... | |
| static SDValue | getMaskNode (SDValue Mask, MVT MaskVT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl) |
Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics. More... | |
| static unsigned | convertBitVectorToUnsiged (const SmallBitVector &Zeroable) |
| static SDValue | lowerVectorShuffleToEXPAND (const SDLoc &DL, MVT VT, const SmallBitVector &Zeroable, ArrayRef< int > Mask, SDValue &V1, SDValue &V2, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | lowerVectorShuffleWithUNPCK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| static SDValue | lowerVectorShuffleAsBitMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SelectionDAG &DAG) |
| Try to emit a bitmask instruction for a shuffle. More... | |
| static SDValue | lowerVectorShuffleAsBitBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Try to emit a blend instruction for a shuffle using bit math. More... | |
| static SDValue | lowerVectorShuffleAsBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Original, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to emit a blend instruction for a shuffle. More... | |
| static SDValue | lowerVectorShuffleAsBlendAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Try to lower as a blend of elements from two inputs followed by a single-input permutation. More... | |
| static SDValue | lowerVectorShuffleAsDecomposedShuffleBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Generic routine to decompose a shuffle and blend into indepndent blends and permutes. More... | |
| static int | matchVectorShuffleAsRotate (SDValue &V1, SDValue &V2, ArrayRef< int > Mask) |
| Try to lower a vector shuffle as a rotation. More... | |
| static int | matchVectorShuffleAsByteRotate (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask) |
| Try to lower a vector shuffle as a byte rotation. More... | |
| static SDValue | lowerVectorShuffleAsByteRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | lowerVectorShuffleAsRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower a vector shuffle as a dword/qword rotation. More... | |
| static int | matchVectorShuffleAsShift (MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget) |
| Try to lower a vector shuffle as a bit shift (shifts in zeros). More... | |
| static SDValue | lowerVectorShuffleAsShift (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | lowerVectorShuffleWithSSE4A (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SelectionDAG &DAG) |
| Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ. More... | |
| static SDValue | lowerVectorShuffleAsSpecificZeroOrAnyExtend (const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower a vector shuffle as a zero or any extension. More... | |
| static SDValue | lowerVectorShuffleAsZeroOrAnyExtend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower a vector shuffle as a zero extension on any microarch. More... | |
| static SDValue | getScalarValueForVectorElement (SDValue V, int Idx, SelectionDAG &DAG) |
| Try to get a scalar value for a specific element of a vector. More... | |
| static bool | isShuffleFoldableLoad (SDValue V) |
| Helper to test for a load that can be folded with x86 shuffles. More... | |
| static SDValue | lowerVectorShuffleAsElementInsertion (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower insertion of a single element into a zero vector. More... | |
| static SDValue | lowerVectorShuffleAsTruncBroadcast (const SDLoc &DL, MVT VT, SDValue V0, int BroadcastIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements. More... | |
| static SDValue | lowerVectorShuffleAsBroadcast (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower broadcast of a single element. More... | |
| static bool | matchVectorShuffleAsInsertPS (SDValue &V1, SDValue &V2, unsigned &InsertPSMask, const SmallBitVector &Zeroable, ArrayRef< int > Mask, SelectionDAG &DAG) |
| static SDValue | lowerVectorShuffleAsInsertPS (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SelectionDAG &DAG) |
| static SDValue | lowerVectorShuffleAsPermuteAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction. More... | |
| static SDValue | lowerV2F64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 2-lane 64-bit floating point shuffles. More... | |
| static SDValue | lowerV2I64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 2-lane 64-bit integer shuffles. More... | |
| static bool | isSingleSHUFPSMask (ArrayRef< int > Mask) |
| Test whether this can be lowered with a single SHUFPS instruction. More... | |
| static SDValue | lowerVectorShuffleWithSHUFPS (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| Lower a vector shuffle using the SHUFPS instruction. More... | |
| static SDValue | lowerV4F32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower 4-lane 32-bit floating point shuffles. More... | |
| static SDValue | lowerV4I32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower 4-lane i32 vector shuffles. More... | |
| static SDValue | lowerV8I16GeneralSingleInputVectorShuffle (const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part. More... | |
| static SDValue | lowerVectorShuffleAsBlendOfPSHUFBs (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) |
| Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used. More... | |
| static SDValue | lowerV8I16VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Generic lowering of 8-lane i16 shuffles. More... | |
| static int | canLowerByDroppingEvenElements (ArrayRef< int > Mask, bool IsSingleInput) |
| Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped. More... | |
| static SDValue | lowerV16I8VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Generic lowering of v16i8 shuffles. More... | |
| static SDValue | lower128BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Dispatching routine to lower various 128-bit x86 vector shuffles. More... | |
| static SDValue | splitAndLowerVectorShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Generic routine to split vector shuffle into half-sized shuffles. More... | |
| static SDValue | lowerVectorShuffleAsSplitOrBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Either split a vector in halves or decompose the shuffles and the blend. More... | |
| static SDValue | lowerVectorShuffleAsLanePermuteAndBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
| Lower a vector shuffle crossing multiple 128-bit lanes as a permutation and blend of those lanes. More... | |
| static SDValue | lowerV2X128VectorShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering 2-lane 128-bit shuffles. More... | |
| static SDValue | lowerVectorShuffleByMerging128BitLanes (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane. More... | |
| static SDValue | lowerVectorShuffleWithUndefHalf (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Lower shuffles where an entire half of a 256-bit vector is UNDEF. More... | |
| static bool | isShuffleMaskInputInPlace (int Input, ArrayRef< int > Mask) |
| Test whether the specified input (0 or 1) is in-place blended by the given mask. More... | |
| static SDValue | lowerShuffleAsRepeatedMaskAndLanePermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes. More... | |
| static bool | matchVectorShuffleWithSHUFPD (MVT VT, SDValue &V1, SDValue &V2, unsigned &ShuffleImm, ArrayRef< int > Mask) |
| static SDValue | lowerVectorShuffleWithSHUFPD (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| static SDValue | lowerVectorShuffleWithPERMV (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| static SDValue | lowerV4F64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 4-lane 64-bit floating point shuffles. More... | |
| static SDValue | lowerV4I64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 4-lane 64-bit integer shuffles. More... | |
| static SDValue | lowerV8F32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 32-bit floating point shuffles. More... | |
| static SDValue | lowerV8I32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 32-bit integer shuffles. More... | |
| static SDValue | lowerV16I16VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 16-lane 16-bit integer shuffles. More... | |
| static SDValue | lowerV32I8VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 32-lane 8-bit integer shuffles. More... | |
| static SDValue | lower256BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| High-level routine to lower various 256-bit x86 vector shuffles. More... | |
| static SDValue | lowerV4X128VectorShuffle (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
| Try to lower a vector shuffle as a 128-bit shuffles. More... | |
| static SDValue | lowerV8F64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 64-bit floating point shuffles. More... | |
| static SDValue | lowerV16F32VectorShuffle (SDLoc DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 16-lane 32-bit floating point shuffles. More... | |
| static SDValue | lowerV8I64VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 8-lane 64-bit integer shuffles. More... | |
| static SDValue | lowerV16I32VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 16-lane 32-bit integer shuffles. More... | |
| static SDValue | lowerV32I16VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 32-lane 16-bit integer shuffles. More... | |
| static SDValue | lowerV64I8VectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, const SmallBitVector &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle lowering of 64-lane 8-bit integer shuffles. More... | |
| static SDValue | lower512BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const SmallBitVector &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| High-level routine to lower various 512-bit x86 vector shuffles. More... | |
| static SDValue | lower1BitVectorShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | canonicalizeShuffleMaskWithCommute (ArrayRef< int > Mask) |
| Helper function that returns true if the shuffle mask should be commuted to improve canonicalization. More... | |
| static SDValue | lowerVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Top-level lowering for x86 vector shuffles. More... | |
| static SDValue | lowerVSELECTtoVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to lower a VSELECT instruction to a vector shuffle. More... | |
| static SDValue | LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerSCALAR_TO_VECTOR (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | areOnlyUsersOf (SDNode *N, ArrayRef< SDValue > ValidUsers) |
| static SDValue | LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | GetTLSADDR (SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LocalDynamic=false) |
| static SDValue | LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT) |
| static SDValue | LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT) |
| static SDValue | LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool is64Bit) |
| static SDValue | LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC) |
| static SDValue | LowerShiftParts (SDValue Op, SelectionDAG &DAG) |
| Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount. More... | |
| static SDValue | lowerUINT_TO_FP_v2i32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDLoc &DL) |
| static SDValue | lowerUINT_TO_FP_vXi32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerAVXExtend (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerZERO_EXTEND_AVX512 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerANY_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerZERO_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | truncateVectorCompareWithPACKSS (EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Helper to recursively truncate vector elements in half with PACKSS. More... | |
| static SDValue | LowerTruncateVecI1 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerFP_EXTEND (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerFABSorFNEG (SDValue Op, SelectionDAG &DAG) |
| The only differences between FABS and FNEG are the mask and the logic op. More... | |
| static SDValue | LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerFGETSIGN (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerVectorAllZeroTest (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | hasNonFlagsUse (SDValue Op) |
return true if Op has a use that doesn't just read flags. More... | |
| static SDValue | EmitKTEST (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | getSETCC (X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl, SelectionDAG &DAG) |
| Helper for creating a X86ISD::SETCC node. More... | |
| static SDValue | getBitTestCondition (SDValue Src, SDValue BitNo, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG) |
Create a BT (Bit Test) node - Test bit BitNo in Src and set condition according to equal/not-equal condition code CC. More... | |
| static SDValue | LowerAndToBT (SDValue And, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG) |
| Result of 'and' is compared against zero. Change to a BT node if possible. More... | |
| static SDValue | LowerTruncateToBT (SDValue Op, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG) |
| static int | translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1) |
| Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs. More... | |
| static SDValue | Lower256IntVSETCC (SDValue Op, SelectionDAG &DAG) |
| Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back. More... | |
| static SDValue | LowerBoolVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerIntVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG) |
| static SDValue | ChangeVSETULTtoVSETULE (const SDLoc &dl, SDValue Op1, SelectionDAG &DAG) |
Try to turn a VSETULT into a VSETULE by modifying its second operand Op1. More... | |
| static SDValue | LowerVSETCC (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | isX86LogicalCmp (SDValue Op) |
| Return true if opcode is a X86 logical comparison. More... | |
| static bool | isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG) |
| static SDValue | LowerSIGN_EXTEND_AVX512 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerEXTEND_VECTOR_INREG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerSIGN_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerTruncatingStore (SDValue StOp, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerExtended1BitVectorLoad (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerExtendedLoad (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | isAndOrOfSetCCs (SDValue Op, unsigned &Opc) |
| Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR. More... | |
| static bool | isXor1OfSetCC (SDValue Op) |
| Return true if node is an ISD::XOR of a X86ISD::SETCC and 1 and that the SETCC node has a single use. More... | |
| static SDValue | LowerVACOPY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | getTargetVShiftByConstNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG) |
| Handle vector element shifts where the shift amount is a constant. More... | |
| static SDValue | getTargetVShiftNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, SDValue ShAmt, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Handle vector element shifts where the shift amount may or may not be a constant. More... | |
| static SDValue | getVectorMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics. More... | |
| static SDValue | getScalarMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Creates an SDNode for a predicated scalar operation. More... | |
| static int | getSEHRegistrationNodeSize (const Function *Fn) |
| static SDValue | recoverFramePointer (SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP) |
| When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP. More... | |
| static SDValue | LowerINTRINSIC_WO_CHAIN (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | getGatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget) |
| static SDValue | getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget) |
| static SDValue | getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget) |
| static void | getExtendedControlRegister (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results) |
| Handles the lowering of builtin intrinsic that return the value of the extended control register. More... | |
| static void | getReadPerformanceCounter (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results) |
| Handles the lowering of builtin intrinsics that read performance monitor counters (x86_rdpmc). More... | |
| static void | getReadTimeStampCounter (SDNode *N, const SDLoc &DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results) |
| Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp). More... | |
| static SDValue | LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | MarkEHRegistrationNode (SDValue Op, SelectionDAG &DAG) |
| static SDValue | MarkEHGuard (SDValue Op, SelectionDAG &DAG) |
| static SDValue | EmitTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG) |
| Emit Truncating Store with signed or unsigned saturation. More... | |
| static SDValue | EmitMaskedTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG) |
| Emit Masked Truncating Store with signed or unsigned saturation. More... | |
| static SDValue | LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerVectorCTLZ_AVX512 (SDValue Op, SelectionDAG &DAG) |
| Lower a vector CTLZ using native supported vector CTLZ instruction. More... | |
| static SDValue | LowerVectorCTLZInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerVectorCTLZ (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCTLZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCTTZ (SDValue Op, SelectionDAG &DAG) |
| static SDValue | Lower256IntArith (SDValue Op, SelectionDAG &DAG) |
| Break a 256-bit integer operation into two new 128-bit ones and then concatenate the result back. More... | |
| static SDValue | Lower512IntArith (SDValue Op, SelectionDAG &DAG) |
| Break a 512-bit integer operation into two new 256-bit ones and then concatenate the result back. More... | |
| static SDValue | LowerADD (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerSUB (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerMINMAX (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerMUL (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMULH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMUL_LOHI (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static bool | SupportedVectorShiftWithImm (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode) |
| static bool | SupportedVectorShiftWithBaseAmnt (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode) |
| static bool | SupportedVectorVarShift (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode) |
| static SDValue | LowerScalarImmediateShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerScalarVariableShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | LowerShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerRotate (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerXALUO (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerATOMIC_FENCE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCMP_SWAP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerBITCAST (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerHorizontalByteSum (SDValue V, MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Compute the horizontal sum of bytes in V for the elements of VT. More... | |
| static SDValue | LowerVectorCTPOPInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerVectorCTPOPBitmath (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerVectorCTPOP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerCTPOP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerBITREVERSE_XOP (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerBITREVERSE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | lowerAtomicArithWithLOCK (SDValue N, SelectionDAG &DAG) |
| static SDValue | lowerAtomicArith (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Lower atomic_load_ops into LOCK-prefixed operations. More... | |
| static SDValue | LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerADDC_ADDE_SUBC_SUBE (SDValue Op, SelectionDAG &DAG) |
| static SDValue | LowerFSINCOS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | ExtendToType (SDValue InOp, MVT NVT, SelectionDAG &DAG, bool FillWithZeroes=false) |
| Widen a vector input to a vector of NVT. More... | |
| static SDValue | LowerMSCATTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMLOAD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMSTORE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static SDValue | LowerMGATHER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| static MachineBasicBlock * | emitXBegin (MachineInstr &MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII) |
| Utility function to emit xbegin specifying the start of an RTM region. More... | |
| static MachineBasicBlock * | emitPCMPSTRM (MachineInstr &MI, MachineBasicBlock *BB, const TargetInstrInfo *TII) |
| static MachineBasicBlock * | emitPCMPSTRI (MachineInstr &MI, MachineBasicBlock *BB, const TargetInstrInfo *TII) |
| static MachineBasicBlock * | emitWRPKRU (MachineInstr &MI, MachineBasicBlock *BB, const X86Subtarget &Subtarget) |
| static MachineBasicBlock * | emitRDPKRU (MachineInstr &MI, MachineBasicBlock *BB, const X86Subtarget &Subtarget) |
| static MachineBasicBlock * | emitMonitor (MachineInstr &MI, MachineBasicBlock *BB, const X86Subtarget &Subtarget, unsigned Opc) |
| static bool | checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI) |
| static bool | isCMOVPseudo (MachineInstr &MI) |
| static bool | matchUnaryVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, bool FloatDomain, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT) |
| static bool | matchUnaryPermuteVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, bool FloatDomain, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) |
| static bool | matchBinaryVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, bool FloatDomain, SDValue &V1, SDValue &V2, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, bool IsUnary) |
| static bool | matchBinaryPermuteVectorShuffle (MVT MaskVT, ArrayRef< int > Mask, bool FloatDomain, SDValue &V1, SDValue &V2, SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) |
| static bool | combineX86ShuffleChain (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Combine an arbitrary chain of shuffles into a single instruction if possible. More... | |
| static bool | combineX86ShufflesConstants (const SmallVectorImpl< SDValue > &Ops, ArrayRef< int > Mask, SDValue Root, bool HasVariableMask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static bool | combineX86ShufflesRecursively (ArrayRef< SDValue > SrcOps, int SrcOpIndex, SDValue Root, ArrayRef< int > RootMask, int Depth, bool HasVariableMask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Fully generic combining of x86 shuffle instructions. More... | |
| static SmallVector< int, 4 > | getPSHUFShuffleMask (SDValue N) |
| Get the PSHUF-style mask from PSHUF node. More... | |
| static SDValue | combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| Search for a combinable shuffle across a chain ending in pshufd. More... | |
| static bool | combineRedundantHalfShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| Search for a combinable shuffle across a chain ending in pshuflw or pshufhw. More... | |
| static SDValue | combineTargetShuffle (SDValue N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Try to combine x86 target specific shuffles. More... | |
| static bool | isAddSub (SDNode *N, const X86Subtarget &Subtarget, SDValue &Opnd0, SDValue &Opnd1) |
Returns true iff the shuffle node N can be replaced with ADDSUB operation. More... | |
| static SDValue | combineShuffleToAddSubOrFMAddSub (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
| Try to combine a shuffle into a target-specific add-sub or mul-add-sub node. More... | |
| static SDValue | combineShuffleOfConcatUndef (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | XFormVExtractWithShuffleIntoLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| Check if a vector extract from a target-specific shuffle of a load can be folded into a single element load. More... | |
| static SDValue | combineBitcast (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | matchBinOpReduction (SDNode *Extract, ISD::NodeType BinOp) |
| static bool | detectZextAbsDiff (const SDValue &Select, SDValue &Op0, SDValue &Op1) |
| static SDValue | createPSADBW (SelectionDAG &DAG, const SDValue &Zext0, const SDValue &Zext1, const SDLoc &DL) |
| static SDValue | combineBasicSADPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineExtractVectorElt (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence. More... | |
| static SDValue | combineVSelectWithAllOnesOrZeros (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation. More... | |
| static SDValue | combineSelectOfTwoConstants (SDNode *N, SelectionDAG &DAG) |
| static bool | combineBitcastForMaskedOp (SDValue OrigOp, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | combineSelect (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on SELECT and VSELECT nodes. More... | |
| static SDValue | combineSetCCAtomicArith (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG) |
| Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction. More... | |
| static SDValue | checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC) |
| static bool | checkBoolTestAndOrSetCCCombine (SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd) |
| Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS. More... | |
| static SDValue | combineSetCCEFLAGS (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG) |
Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values. More... | |
| static SDValue | combineCMov (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]. More... | |
| static bool | canReduceVMulWidth (SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode) |
| static SDValue | reduceVMULWidth (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code. More... | |
| static SDValue | combineMul (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Optimize a single multiply with constant into two operations in order to implement it with two cheaper instructions, e.g. More... | |
| static SDValue | combineShiftLeft (SDNode *N, SelectionDAG &DAG) |
| static SDValue | combineShiftRightAlgebraic (SDNode *N, SelectionDAG &DAG) |
| static SDValue | performShiftToAllZeros (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Returns a vector of 0s if the node in input is a vector logical shift by a constant amount which is known to be bigger than or equal to the vector element size in bits. More... | |
| static SDValue | combineShift (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineVectorShift (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineCompareEqual (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends. More... | |
| static SDValue | combineANDXORWithAllOnesIntoANDNP (SDNode *N, SelectionDAG &DAG) |
| Try to fold: (and (xor X, -1), Y) -> (andnp X, Y). More... | |
| static SDValue | WidenMaskArithmetic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | convertIntLogicToFPLogic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| If both input operands of a logic op are being cast from floating point types, try to convert this into a floating point logic node to avoid unnecessary moves from SSE to integer registers. More... | |
| static SDValue | combinePCMPAnd1 (SDNode *N, SelectionDAG &DAG) |
| If this is a PCMPEQ or PCMPGT result that is bitwise-anded with 1 (this is the x86 lowering of a SETCC + ZEXT), replace the 'and' with a shift-right to eliminate loading the vector constant mask value. More... | |
| static SDValue | combineAnd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineLogicBlendIntoPBLENDV (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | lowerX86CmpEqZeroToCtlzSrl (SDValue Op, EVT ExtTy, SelectionDAG &DAG) |
| static SDValue | combineOrCmpEqZeroToCtlzSrl (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineIntegerAbs (SDNode *N, SelectionDAG &DAG) |
| Generate NEG and CMOV for integer abs. More... | |
| static SDValue | foldXorTruncShiftIntoCmp (SDNode *N, SelectionDAG &DAG) |
| Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1) More... | |
| static SDValue | foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1. More... | |
| static SDValue | detectAVGPattern (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL) |
| This function detects the AVG pattern between vectors of unsigned i8/i16, which is c = (a + b + 1) / 2, and replace this operation with the efficient X86ISD::AVG instruction. More... | |
| static SDValue | combineLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static int | getOneTrueElt (SDValue V) |
| If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element. More... | |
| static bool | getParamsForOneTrueMaskedElt (MaskedLoadStoreSDNode *MaskedOp, SelectionDAG &DAG, SDValue &Addr, SDValue &Index, unsigned &Alignment) |
| Given a masked memory load/store operation, return true if it has one mask bit set. More... | |
| static SDValue | reduceMaskedLoadToScalarLoad (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert. More... | |
| static SDValue | combineMaskedLoadConstantMask (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | combineMaskedLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | reduceMaskedStoreToScalarStore (MaskedStoreSDNode *MS, SelectionDAG &DAG) |
| If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store. More... | |
| static SDValue | combineMaskedStore (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineStore (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static bool | isHorizontalBinOp (SDValue &LHS, SDValue &RHS, bool IsCommutative) |
| Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS. More... | |
| static SDValue | combineFaddFsub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on floating-point adds/subs. More... | |
| static SDValue | combineTruncatedArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDLoc &DL) |
| Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen. More... | |
| static SDValue | combineVectorTruncationWithPACKUS (SDNode *N, SelectionDAG &DAG, SmallVector< SDValue, 8 > &Regs) |
| Truncate a group of v4i32 into v16i8/v8i16 using X86ISD::PACKUS. More... | |
| static SDValue | combineVectorTruncationWithPACKSS (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG, SmallVector< SDValue, 8 > &Regs) |
| Truncate a group of v4i32 into v8i16 using X86ISD::PACKSS. More... | |
| static SDValue | combineVectorTruncation (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| This function transforms truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations. More... | |
| static SDValue | combineVectorSignBitsTruncation (SDNode *N, SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| This function transforms vector truncation of 'all or none' bits values. More... | |
| static SDValue | combineTruncate (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | isFNEG (SDNode *N) |
Returns the negated value if the node N flips sign of FP value. More... | |
| static SDValue | combineFneg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on floating point negations. More... | |
| static SDValue | lowerX86FPLogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineXor (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static bool | isNullFPScalarOrVectorConst (SDValue V) |
| static SDValue | getNullFPConstForNullVal (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value. More... | |
| static SDValue | combineFAndFNotToFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineFAnd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on X86ISD::FAND nodes. More... | |
| static SDValue | combineFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on X86ISD::FANDN nodes. More... | |
| static SDValue | combineFOr (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes. More... | |
| static SDValue | combineFMinFMax (SDNode *N, SelectionDAG &DAG) |
| Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes. More... | |
| static SDValue | combineFMinNumFMaxNum (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineBT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | combineSignExtendInReg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | promoteExtBeforeAdd (SDNode *Ext, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| sext(add_nsw(x, C)) –> add(sext(x), C_sext) zext(add_nuw(x, C)) –> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode. More... | |
| static SDValue | getDivRem8 (SDNode *N, SelectionDAG &DAG) |
| (i8,i32 {s/z}ext ({s/u}divrem (i8 x, i8 y)) -> (i8,i32 ({s/u}divrem_sext_hreg (i8 x, i8 y) This exposes the {s/z}ext to the sdivrem lowering, so that it directly extends from AH (which we otherwise need to do contortions to access). More... | |
| static SDValue | combineToExtendVectorInReg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Convert a SEXT or ZEXT of a vector to a SIGN_EXTEND_VECTOR_INREG or ZERO_EXTEND_VECTOR_INREG, this requires the splitting (or concatenating with UNDEFs) of the input to vectors of the same size as the target type which then extends the lowest elements. More... | |
| static SDValue | combineSext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineFMA (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineZext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineSetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Optimize x == -y –> x+y == 0 x != -y –> x+y != 0. More... | |
| static SDValue | combineGatherScatter (SDNode *N, SelectionDAG &DAG) |
| static SDValue | MaterializeSETB (const SDLoc &DL, SDValue EFLAGS, SelectionDAG &DAG, MVT VT) |
| static SDValue | combineX86SetCC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineBrCond (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| Optimize branch condition evaluation. More... | |
| static SDValue | combineVectorCompareAndMaskUnaryOp (SDNode *N, SelectionDAG &DAG) |
| static SDValue | combineUIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineSIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineADC (SDNode *N, SelectionDAG &DAG, X86TargetLowering::DAGCombinerInfo &DCI) |
| static SDValue | OptimizeConditionalInDecrement (SDNode *N, SelectionDAG &DAG) |
| fold (add Y, (sete X, 0)) -> adc 0, Y (add Y, (setne X, 0)) -> sbb -1, Y (sub (sete X, 0), Y) -> sbb 0, Y (sub (setne X, 0), Y) -> adc -1, Y More... | |
| static SDValue | combineLoopSADPattern (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineAdd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineSub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static SDValue | combineVSZext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
| static SDValue | combineLockSub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| Canonicalize (LSUB p, 1) -> (LADD p, -1). More... | |
| static SDValue | combineTestM (SDNode *N, SelectionDAG &DAG) |
| static SDValue | combineVectorCompare (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
| static bool | matchAsm (StringRef S, ArrayRef< const char * > Pieces) |
| static bool | clobbersFlagRegisters (const SmallVector< StringRef, 4 > &AsmPieces) |
| static bool | isGRClass (const TargetRegisterClass &RC) |
Check if RC is a general purpose register class. More... | |
| static bool | isFRClass (const TargetRegisterClass &RC) |
Check if RC is a vector register class. More... | |
Variables | |
| static cl::opt< bool > | ExperimentalVectorWideningLegalization ("x86-experimental-vector-widening-legalization", cl::init(false), cl::desc("Enable an experimental vector type legalization through widening ""rather than promotion."), cl::Hidden) |
| #define DEBUG_TYPE "x86-isel" |
Definition at line 63 of file X86ISelLowering.cpp.
| enum ShrinkMode |
Different mul shrinking modes.
| Enumerator | |
|---|---|
| MULS8 | |
| MULU8 | |
| MULS16 | |
| MULU16 | |
Definition at line 29931 of file X86ISelLowering.cpp.
| enum StructReturnType |
CallIsStructReturn - Determines whether a call uses struct return semantics.
| Enumerator | |
|---|---|
| NotStructReturn | |
| RegStructReturn | |
| StackStructReturn | |
Definition at line 2569 of file X86ISelLowering.cpp.
Definition at line 13958 of file X86ISelLowering.cpp.
References llvm::all_of(), E, llvm::SDValue::getNode(), I, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().
Referenced by LowerINSERT_SUBVECTOR().
|
static |
Determines whether a function uses struct return semantics.
Definition at line 2589 of file X86ISelLowering.cpp.
References llvm::SmallVectorBase::empty(), fuzzer::Flags, llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSRet(), NotStructReturn, RegStructReturn, and StackStructReturn.
|
static |
Definition at line 6634 of file X86ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), getUnderlyingExtractedFromVec(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), i, llvm::ISD::INSERT_VECTOR_ELT, llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::BitmaskEnumDetail::Mask(), llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::ISD::UNDEF.
|
static |
Definition at line 2575 of file X86ISelLowering.cpp.
References llvm::SmallVectorBase::empty(), fuzzer::Flags, llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSRet(), NotStructReturn, RegStructReturn, and StackStructReturn.
|
static |
Return true if the calling convention is one that we can guarantee TCO for.
Definition at line 2616 of file X86ISelLowering.cpp.
References llvm::CallingConv::Fast, llvm::CallingConv::GHC, llvm::CallingConv::HHVM, llvm::CallingConv::HiPE, and llvm::CallingConv::X86_RegCall.
Referenced by mayTailCallThisCC(), and shouldGuaranteeTCO().
Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped.
This handles shuffles which take every Nth element where N is a power of two. Example shuffle masks:
N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24
Any of these lanes can of course be undef.
This routine only supports N <= 3. FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here for larger N.
Definition at line 11025 of file X86ISelLowering.cpp.
References llvm::array_lengthof(), assert(), i, llvm::isPowerOf2_32(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I8VectorShuffle().
Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.
Definition at line 13279 of file X86ISelLowering.cpp.
References assert(), i, and llvm::ArrayRef< T >::size().
Referenced by combineX86ShufflesRecursively(), and lowerVectorShuffle().
|
static |
Definition at line 29933 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::ComputeNumSignBits(), llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SDNode::getNumOperands(), llvm::APInt::getNumSignBits(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), i, llvm::MVT::i16, llvm::MVT::i8, llvm::tgtok::IntVal, llvm::APInt::isNegative(), fuzzer::min(), MULS16, MULS8, MULU16, MULU8, llvm::SDNode::op_values(), and llvm::ISD::ZERO_EXTEND.
Referenced by reduceVMULWidth().
|
static |
Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.
Appends the mask for wider elements in WidenedMask if valid. Otherwise leaves it in an unspecified state.
NOTE: This must handle normal vector shuffle masks and target vector shuffle masks. The latter have the special property of a '-2' representing a zero-ed lane of a vector.
Definition at line 4554 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::assign(), i, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineX86ShufflesRecursively(), lowerV2X128VectorShuffle(), lowerV4X128VectorShuffle(), and lowerVectorShuffle().
|
static |
Try to turn a VSETULT into a VSETULE by modifying its second operand Op1.
If non-trivial (for example because it's not constant) return an empty value.
Definition at line 16652 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), i, llvm::ConstantSDNode::isOpaque(), and llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back().
Referenced by LowerVSETCC().
|
static |
|
static |
Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
Match: (X86or (X86setcc) (X86setcc)) (X86cmp (and (X86setcc) (X86setcc)), 0)
Definition at line 29676 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::AND, llvm::X86ISD::CMP, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::isNullConstant(), LLVM_FALLTHROUGH, llvm::ISD::OR, llvm::X86ISD::OR, and llvm::X86ISD::SETCC.
Referenced by combineCMov().
|
static |
Definition at line 29558 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::X86ISD::CMOV, llvm::X86ISD::CMP, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::dyn_cast(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::SDValue::getResNo(), llvm::ConstantSDNode::getZExtValue(), llvm::SDNode::hasAnyUseOfValue(), llvm::isOneConstant(), LLVM_FALLTHROUGH, llvm::X86ISD::RDRAND, llvm::X86ISD::RDSEED, llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSetCCEFLAGS().
|
static |
Definition at line 34054 of file X86ISelLowering.cpp.
References llvm::SmallVectorTemplateCommon< T >::begin(), llvm::count(), llvm::SmallVectorTemplateCommon< T >::end(), and llvm::SmallVectorTemplateCommon< T >::size().
Referenced by llvm::X86TargetLowering::ExpandInlineAsm().
|
static |
Definition at line 33516 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86::COND_B, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::MVT::i8, llvm::X86::isZeroNode(), llvm::X86ISD::SETCC_CARRY, and llvm::SDValue::use_empty().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 33653 of file X86ISelLowering.cpp.
References combineLoopSADPattern(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::SDNodeFlags::hasVectorReduction(), isHorizontalBinOp(), N, OptimizeConditionalInDecrement(), llvm::MVT::v16i16, llvm::MVT::v4i32, llvm::MVT::v8i16, and llvm::MVT::v8i32.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 30735 of file X86ISelLowering.cpp.
References llvm::X86ISD::BEXTR, combineANDXORWithAllOnesIntoANDNP(), combineCompareEqual(), combinePCMPAnd1(), combineX86ShufflesRecursively(), convertIntLogicToFPLogic(), llvm::countPopulation(), llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::X86Subtarget::hasBMI(), llvm::X86Subtarget::hasTBM(), llvm::MVT::i32, llvm::MVT::i64, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isMask_64(), llvm::EVT::isVector(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ISD::SRA, and llvm::ISD::SRL.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
Definition at line 30542 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllOnes(), llvm::SDValue::isUndef(), peekThroughBitcasts(), std::swap(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, llvm::MVT::v4i64, llvm::MVT::v8i64, and llvm::ISD::XOR.
Referenced by combineAnd().
|
static |
Definition at line 28579 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::BITCAST, createPSADBW(), detectZextAbsDiff(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSE2(), i, llvm::MVT::i32, llvm::EVT::isSimple(), llvm::Log2_32(), llvm::BitmaskEnumDetail::Mask(), matchBinOpReduction(), and llvm::ISD::VSELECT.
Referenced by combineExtractVectorElt().
|
static |
Definition at line 28392 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i32, llvm::isNullConstant(), llvm::X86ISD::MMX_MOVW2D, llvm::ISD::OR, llvm::MVT::v2i32, llvm::MVT::x86mmx, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 28987 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::BITCAST, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::MVT::isInteger(), LLVM_FALLTHROUGH, llvm::X86ISD::PALIGNR, llvm::X86ISD::SHUF128, and llvm::X86ISD::VALIGN.
Referenced by combineSelect().
|
static |
Optimize branch condition evaluation.
Definition at line 33354 of file X86ISelLowering.cpp.
References llvm::X86ISD::BRCOND, combineSetCCEFLAGS(), fuzzer::Flags, llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getVTList(), and llvm::MVT::i8.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 32807 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), llvm::APInt::getLowBitsSet(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and llvm::Log2_32().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].
Definition at line 29725 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::BSF, llvm::X86ISD::BSR, checkBoolTestAndOrSetCCCombine(), llvm::ARMISD::CMOV, llvm::X86ISD::CMOV, llvm::X86ISD::CMP, combineSetCCEFLAGS(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::X86::COND_E, llvm::X86::COND_NE, llvm::MVT::f80, fuzzer::Flags, llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDNode::getVTList(), hasFPCMov(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::SelectionDAG::isKnownNeverZero(), llvm::ISD::MUL, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SHL, llvm::X86ISD::SUB, std::swap(), llvm::SDValue::use_empty(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends.
Likewise for OR -> CMPNEQSS.
Definition at line 30441 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BR_CC, llvm::ISD::BRCOND, llvm::X86ISD::CMP, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_P, llvm::ISD::CopyToReg, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FSETCC, llvm::X86ISD::FSETCCM, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), isAndOrOfSetCCs(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SELECT, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::MVT::v2f64, llvm::MVT::v4f32, and llvm::ISD::ZERO_EXTEND.
Referenced by combineAnd(), and combineOr().
|
static |
Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.
For i686, the best sequence is apparently storing the value and loading scalars back, while for x64 we should use 64-bit extracts and shifts.
Definition at line 28650 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SmallVectorTemplateCommon< T >::begin(), llvm::ISD::BITCAST, combineBasicSADPattern(), llvm::SelectionDAG::CreateStackTemporary(), llvm::SmallVectorTemplateCommon< T >::end(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SDValue::getResNo(), llvm::TargetLoweringBase::getShiftAmountTy(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::TargetLoweringBase::getVectorIdxTy(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), i, llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::TargetLoweringBase::isOperationLegal(), llvm::X86ISD::MMX_MOVD2W, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::x86mmx, XFormVExtractWithShuffleIntoLoad(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on floating-point adds/subs.
Definition at line 32169 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasFp256(), llvm::X86Subtarget::hasSSE3(), isHorizontalBinOp(), llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, and llvm::MVT::v8f32.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on X86ISD::FAND nodes.
Definition at line 32680 of file X86ISelLowering.cpp.
References combineFAndFNotToFAndn(), getNullFPConstForNullVal(), llvm::SDNode::getOperand(), and lowerX86FPLogicOp().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 32651 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FANDN, llvm::X86ISD::FXOR, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasSSE1(), and llvm::X86Subtarget::hasSSE2().
Referenced by combineFAnd().
|
static |
Do target-specific dag combines on X86ISD::FANDN nodes.
Definition at line 32697 of file X86ISelLowering.cpp.
References getNullFPConstForNullVal(), llvm::SDNode::getOperand(), isNullFPScalarOrVectorConst(), and lowerX86FPLogicOp().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 33088 of file X86ISelLowering.cpp.
References A, assert(), B, llvm::MVT::f32, llvm::MVT::f64, llvm::ISD::FMA, llvm::X86ISD::FMADD, llvm::X86ISD::FMADD_RND, llvm::X86ISD::FMADDS1_RND, llvm::X86ISD::FMADDS3_RND, llvm::X86ISD::FMSUB, llvm::X86ISD::FMSUB_RND, llvm::X86ISD::FMSUBS1_RND, llvm::X86ISD::FMSUBS3_RND, llvm::X86ISD::FNMADD, llvm::X86ISD::FNMADD_RND, llvm::X86ISD::FNMADDS1_RND, llvm::X86ISD::FNMADDS3_RND, llvm::X86ISD::FNMSUB, llvm::X86ISD::FNMSUB_RND, llvm::X86ISD::FNMSUBS1_RND, llvm::X86ISD::FNMSUBS3_RND, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAnyFMA(), isFNEG(), and llvm::TargetLoweringBase::isTypeLegal().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
Definition at line 32731 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::FMAX, llvm::X86ISD::FMAXC, llvm::X86ISD::FMIN, llvm::X86ISD::FMINC, llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTarget(), llvm::SDNode::getValueType(), llvm_unreachable, llvm::TargetMachine::Options, and llvm::TargetOptions::UnsafeFPMath.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 32751 of file X86ISelLowering.cpp.
References llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FMAX, llvm::ISD::FMAXNUM, llvm::X86ISD::FMIN, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::EVT::isVector(), llvm::Function::optForMinSize(), llvm::ISD::SELECT, llvm::ISD::SETUO, llvm::X86Subtarget::useSoftFloat(), llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v8f32, and llvm::ISD::VSELECT.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on floating point negations.
Definition at line 32530 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FMADD, llvm::X86ISD::FMADD_RND, llvm::X86ISD::FMSUB, llvm::X86ISD::FMSUB_RND, llvm::ISD::FMUL, llvm::X86ISD::FNMADD, llvm::X86ISD::FNMADD_RND, llvm::X86ISD::FNMSUB, llvm::X86ISD::FNMSUB_RND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantFP(), llvm::SDNode::getFlags(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDNodeFlags::hasNoSignedZeros(), llvm::SDValue::hasOneUse(), isFNEG(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDNode::ops(), and Zero.
Referenced by combineFOr(), combineXor(), and llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
Definition at line 32711 of file X86ISelLowering.cpp.
References assert(), combineFneg(), llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), isFNEG(), isNullFPScalarOrVectorConst(), and lowerX86FPLogicOp().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 33282 of file X86ISelLowering.cpp.
References llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::ISD::SIGN_EXTEND_INREG, and llvm::SelectionDAG::UpdateNodeOperands().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Generate NEG and CMOV for integer abs.
Definition at line 31144 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::CMOV, llvm::X86::COND_GE, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::MVT::i32, llvm::MVT::i8, llvm::EVT::isInteger(), llvm::ISD::SRA, llvm::X86ISD::SUB, and llvm::ISD::XOR.
Referenced by combineXor().
|
static |
Definition at line 31396 of file X86ISelLowering.cpp.
References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::CallingConv::Fast, llvm::MemSDNode::getAddressSpace(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::LoadSDNode::getExtensionType(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorVT(), insert128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), fuzzer::min(), N, llvm::ISD::NON_EXTLOAD, llvm::MVT::Other, Ptr, and llvm::ISD::TokenFactor.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Canonicalize (LSUB p, 1) -> (LADD p, -1).
Definition at line 33791 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::X86ISD::LADD, N, and llvm::MVT::Other.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 30803 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::isBuildVectorAllZeros(), llvm::BitmaskEnumDetail::Mask(), N, llvm::ISD::OR, peekThroughBitcasts(), llvm::ISD::SRA, llvm::ISD::SUB, std::swap(), llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i64, llvm::ISD::VSELECT, llvm::X86ISD::VSRAI, X, llvm::ISD::XOR, and Y.
Referenced by combineOr().
|
static |
Definition at line 33582 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::BITCAST, createPSADBW(), detectZextAbsDiff(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasBWI(), llvm::MVT::i32, llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::ISD::TRUNCATE, and llvm::ISD::VSELECT.
Referenced by combineAdd().
|
static |
Definition at line 31577 of file X86ISelLowering.cpp.
References assert(), combineMaskedLoadConstantMask(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, llvm::MaskedLoadStoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MaskedLoadSDNode::getSrc0(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), i, llvm::MVT::i1, llvm::MaskedLoadSDNode::isExpandingLoad(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::BitmaskEnumDetail::Mask(), N, llvm::ISD::NON_EXTLOAD, reduceMaskedLoadToScalarLoad(), llvm::ISD::SEXTLOAD, and llvm::X86ISD::VSEXT.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 31535 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::MaskedLoadStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::SelectionDAG::getLoad(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelect(), llvm::MaskedLoadSDNode::getSrc0(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isNullConstant(), and llvm::SDValue::isUndef().
Referenced by combineMaskedLoad().
|
static |
Definition at line 31698 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::MaskedLoadStoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MaskedStoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), i, llvm::MVT::i1, llvm::MaskedStoreSDNode::isCompressingStore(), llvm::isPowerOf2_32(), llvm::MaskedStoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::BitmaskEnumDetail::Mask(), N, and reduceMaskedStoreToScalarStore().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Optimize a single multiply with constant into two operations in order to implement it with two cheaper instructions, e.g.
LEA + SHL, LEA + LEA.
Definition at line 30146 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::SDNode::hasOneUse(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm::isPowerOf2_64(), llvm::EVT::isVector(), llvm::Log2_64(), llvm::X86ISD::MUL_IMM, llvm::Function::optForMinSize(), reduceVMULWidth(), llvm::ISD::SHL, llvm::ISD::SUB, std::swap(), and llvm::SDNode::use_begin().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 31030 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::tgtok::Bits, combineCompareEqual(), combineLogicBlendIntoPBLENDV(), convertIntLogicToFPLogic(), llvm::dyn_cast(), llvm::SDValue::getConstantOperandVal(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::ConstantSDNode::getSExtValue(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::X86Subtarget::isSHLDSlow(), llvm::Function::optForSize(), llvm::ISD::SHL, llvm::X86ISD::SHLD, llvm::X86ISD::SHRD, llvm::ISD::SRL, llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 30950 of file X86ISelLowering.cpp.
References llvm::EVT::bitsGE(), llvm::MVT::bitsGE(), llvm::X86ISD::CMP, llvm::X86::COND_E, llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::X86Subtarget::getTargetLowering(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::MVT::i32, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::X86TargetLowering::isCtlzFast(), lowerX86CmpEqZeroToCtlzSrl(), N, OR, llvm::ISD::OR, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::MipsISD::Ret, llvm::X86ISD::SETCC, llvm::SmallVectorTemplateCommon< T >::size(), std::swap(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineZext().
|
static |
If this is a PCMPEQ or PCMPGT result that is bitwise-anded with 1 (this is the x86 lowering of a SETCC + ZEXT), replace the 'and' with a shift-right to eliminate loading the vector constant mask value.
This relies on the fact that a PCMP always creates an all-ones or all-zeros bitmask per element.
Definition at line 30703 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i8, llvm::ISD::isConstantSplatVector(), llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, peekThroughBitcasts(), and llvm::X86ISD::VSRLI.
Referenced by combineAnd().
|
static |
Search for a combinable shuffle across a chain ending in pshufd.
We walk up the chain and look for a combinable shuffle, skipping over shuffles that we could hoist this shuffle's transformation past without altering anything.
Definition at line 27563 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::SmallVectorBase::empty(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPSHUFShuffleMask(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::MVT::i16, llvm::MVT::i8, llvm::SDNode::isOnlyUserOf(), LLVM_FALLTHROUGH, llvm_unreachable, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by combineTargetShuffle().
|
static |
Search for a combinable shuffle across a chain ending in pshuflw or pshufhw.
We walk up the chain, skipping shuffles of the other half and looking through shuffles which switch halves trying to find a shuffle of the same pair of dwords.
Definition at line 27697 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPSHUFShuffleMask(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::hasOneUse(), llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, and llvm::MVT::v8i16.
Referenced by combineTargetShuffle().
|
static |
Do target-specific dag combines on SELECT and VSELECT nodes.
Definition at line 29096 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), assert(), combineBitcastForMaskedOp(), combineSelectOfTwoConstants(), combineVSelectWithAllOnesOrZeros(), llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), E, llvm::MVT::f128, llvm::MVT::f32, llvm::MVT::f80, llvm::X86ISD::FMAX, llvm::X86ISD::FMIN, llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasVLX(), I, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i8, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SelectionDAG::isEqualTo(), llvm::EVT::isFloatingPoint(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::SelectionDAG::isKnownNeverZero(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::TargetMachine::Options, Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::X86ISD::SHRUNKBLEND, llvm::ISD::SIGN_EXTEND, llvm::TargetLowering::SimplifyDemandedBits(), llvm::ISD::SUB, llvm::X86ISD::SUBUS, std::swap(), llvm::TargetOptions::UnsafeFPMath, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v2f32, llvm::MVT::v32i8, llvm::MVT::v8i16, llvm::ISD::VSELECT, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 28882 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::MUL, llvm::ISD::SETCC, llvm::ISD::SHL, std::swap(), llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSelect().
|
static |
Optimize x == -y –> x+y == 0 x != -y –> x+y != 0.
Definition at line 33217 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCSwappedOperands(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::ISD::isBuildVectorAllZeros(), llvm::isNullConstant(), llvm::ISD::isSignedIntSetCC(), LowerVSETCC(), llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, std::swap(), llvm::MVT::v4f32, and llvm::MVT::v4i32.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction.
Note that this is only legal for some op/cc combinations.
Definition at line 29476 of file X86ISelLowering.cpp.
References llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_SUB, llvm::X86ISD::CMP, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NS, llvm::X86::COND_S, llvm::dyn_cast(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDValue::hasOneUse(), lowerAtomicArithWithLOCK(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and llvm::X86ISD::SUB.
Referenced by combineSetCCEFLAGS().
|
static |
Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values.
Definition at line 29717 of file X86ISelLowering.cpp.
References checkBoolTestSetCCCombine(), and combineSetCCAtomicArith().
Referenced by combineBrCond(), combineCMov(), and combineX86SetCC().
|
static |
Definition at line 33054 of file X86ISelLowering.cpp.
References combineToExtendVectorInReg(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getConstant(), getDivRem8(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX(), llvm::MVT::i1, llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), promoteExtBeforeAdd(), llvm::ISD::SELECT, and WidenMaskArithmetic().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 30380 of file X86ISelLowering.cpp.
References combineShiftLeft(), combineShiftRightAlgebraic(), llvm::SDNode::getOpcode(), performShiftToAllZeros(), llvm::ISD::SHL, and llvm::ISD::SRA.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 30236 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::Constant, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::APInt::isIntN(), llvm::EVT::isVector(), llvm::X86ISD::SETCC_CARRY, llvm::APInt::shl(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by combineShift().
|
static |
Definition at line 30296 of file X86ISelLowering.cpp.
References llvm::ISD::Constant, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::integer_valuetypes(), llvm::EVT::isInteger(), llvm::APInt::isNegative(), llvm::EVT::isVector(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, and llvm::ISD::SRA.
Referenced by combineShift().
|
static |
Definition at line 28171 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::BITCAST, combineShuffleOfConcatUndef(), combineShuffleToAddSubOrFMAddSub(), combineTargetShuffle(), combineX86ShufflesRecursively(), EltsFromConsecutiveLoads(), llvm::ISD::FADD, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::ShuffleVectorSDNode::getMask(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), getShuffleScalarElt(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::hasOneUse(), i, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isOperationLegal(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), llvm::ARM_MB::LD, llvm::ISD::MUL, N, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ISD::SUB, and llvm::ISD::VECTOR_SHUFFLE.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 28129 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::MVT::f32, llvm::MVT::f64, llvm::ShuffleVectorSDNode::getMask(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i32, llvm::MVT::i64, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::SDValue::isUndef(), llvm::BitmaskEnumDetail::Mask(), N, and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().
Referenced by combineShuffle().
|
static |
Try to combine a shuffle into a target-specific add-sub or mul-add-sub node.
Definition at line 28102 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADDSUB, llvm::X86ISD::FMADDSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getValueType(), llvm::EVT::is512BitVector(), isAddSub(), and isFMAddSub().
Referenced by combineShuffle().
|
static |
Definition at line 32825 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasInt256(), llvm::ISD::isNormalLoad(), llvm::EVT::isVector(), llvm::ISD::LOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::MVT::v4i32, and llvm::MVT::v4i64.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 33450 of file X86ISelLowering.cpp.
References llvm::X86TargetLowering::BuildFILD(), combineVectorCompareAndMaskUnaryOp(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::MVT::f128, llvm::MVT::f16, llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::X86Subtarget::getTargetLowering(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasDQI(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::ISD::isNON_EXTLoad(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::MemSDNode::isVolatile(), llvm::ISD::LOAD, P, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SINT_TO_FP, llvm::ISD::TRUNCATE, and llvm::X86Subtarget::useSoftFloat().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 31791 of file X86ISelLowering.cpp.
References llvm::TargetLoweringBase::allowsMemoryAccess(), assert(), detectAVGPattern(), extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::CallingConv::Fast, llvm::MemSDNode::getAddressSpace(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::MVT::getStoreSize(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MachinePointerInfo::getWithOffset(), llvm::Function::hasFnAttribute(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), i, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::MVT::integer_valuetypes(), llvm::X86Subtarget::is64Bit(), llvm::ISD::isNormalLoad(), llvm::isPowerOf2_32(), llvm::StoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegalOrCustom(), llvm::TargetLoweringBase::isTypeLegal(), isVolatile(), llvm::MemSDNode::isVolatile(), fuzzer::min(), llvm::MinAlign(), N, llvm::MVT::Other, Ptr, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ISD::TokenFactor, and llvm::X86Subtarget::useSoftFloat().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 33673 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, isHorizontalBinOp(), OptimizeConditionalInDecrement(), llvm::MVT::v16i16, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::MVT::v8i32, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Try to combine x86 target specific shuffles.
Definition at line 27760 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::X86ISD::BLENDI, combineRedundantDWordShuffle(), combineRedundantHalfShuffle(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), D, llvm::X86ISD::FHADD, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), getPSHUFShuffleMask(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), i, llvm::MVT::i32, llvm::MVT::i8, llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::HexagonMCInstrInfo::isFloat(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), isShuffleEquivalent(), isShuffleFoldableLoad(), llvm::SDValue::isUndef(), isUndefOrZero(), llvm::makeArrayRef(), llvm::BitmaskEnumDetail::Mask(), llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, peekThroughBitcasts(), peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, setTargetShuffleZeroElements(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4i32, and llvm::ISD::VECTOR_SHUFFLE.
Referenced by combineShuffle().
|
static |
Definition at line 33811 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), and llvm::X86ISD::TESTM.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Convert a SEXT or ZEXT of a vector to a SIGN_EXTEND_VECTOR_INREG or ZERO_EXTEND_VECTOR_INREG, this requires the splitting (or concatenating with UNDEFs) of the input to vectors of the same size as the target type which then extends the lowest elements.
Definition at line 32955 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSignExtendVectorInReg(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getZeroExtendVectorInReg(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), i, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSext(), and combineZext().
|
static |
Definition at line 32447 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, combineTruncatedArithmetic(), combineVectorSignBitsTruncation(), combineVectorTruncation(), detectAVGPattern(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::X86ISD::MMX_MOVD2W, and llvm::MVT::x86mmx.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen.
e.g. TRUNC( BINOP( X, Y ) ) –> BINOP( TRUNC( X ), TRUNC( Y ) )
Definition at line 32190 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::MVT::i64, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDNode::isOnlyUserOf(), llvm::TargetLoweringBase::isOperationLegal(), llvm::TargetLoweringBase::isOperationLegalOrPromote(), llvm::EVT::isVector(), LLVM_FALLTHROUGH, llvm::ISD::MUL, llvm::ISD::OR, peekThroughOneUseBitcasts(), llvm::ISD::TRUNCATE, and llvm::ISD::XOR.
Referenced by combineTruncate().
|
static |
Definition at line 33419 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::TargetLoweringBase::isOperationLegal(), llvm::EVT::isVector(), P, llvm::SelectionDAG::SignBitIsZero(), llvm::ISD::SINT_TO_FP, llvm::ISD::UINT_TO_FP, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 33825 of file X86ISelLowering.cpp.
References getOnesVector(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), getZeroVector(), llvm::X86ISD::PCMPEQ, and llvm::X86ISD::PCMPGT.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 33373 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::isVector(), and llvm::ISD::SETCC.
Referenced by combineSIntToFP().
|
static |
Definition at line 30399 of file X86ISelLowering.cpp.
References assert(), combineX86ShufflesRecursively(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SDNode::getValueType(), getZeroVector(), llvm::APInt::getZExtValue(), llvm::ISD::isBuildVectorAllZeros(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::APInt::uge(), llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLI, and llvm::APInt::zextOrTrunc().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
This function transforms vector truncation of 'all or none' bits values.
vXi16/vXi32/vXi64 to vXi8/vXi16/vXi32 into X86ISD::PACKSS operations.
Definition at line 32410 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::ComputeNumSignBits(), llvm::SDNode::getOperand(), llvm::MVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::EVT::isSimple(), llvm::EVT::isVector(), and truncateVectorCompareWithPACKSS().
Referenced by combineTruncate().
|
static |
This function transforms truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations.
We do it here because after type legalization the truncation will be translated into a BUILD_VECTOR with each element that is extracted from a vector and then truncated, and it is difficult to do this optimization based on them.
Definition at line 32353 of file X86ISelLowering.cpp.
References combineVectorTruncationWithPACKSS(), combineVectorTruncationWithPACKUS(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::isPowerOf2_32(), llvm::EVT::isSimple(), and llvm::EVT::isVector().
Referenced by combineTruncate().
|
static |
Truncate a group of v4i32 into v8i16 using X86ISD::PACKSS.
Definition at line 32321 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getTargetVShiftNode(), llvm::SDNode::getValueType(), llvm::MVT::i32, llvm::X86ISD::PACKSS, llvm::SmallVectorTemplateCommon< T >::size(), llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::X86ISD::VSHLI, and llvm::X86ISD::VSRAI.
Referenced by combineVectorTruncation().
|
static |
Truncate a group of v4i32 into v16i8/v8i16 using X86ISD::PACKUS.
Definition at line 32266 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::i8, llvm::X86ISD::PACKUS, llvm::SmallVectorTemplateCommon< T >::size(), llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, and llvm::MVT::v8i8.
Referenced by combineVectorTruncation().
|
static |
If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation.
Definition at line 28794 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::ISD::OR, llvm::APIntOps::Or(), llvm::ISD::SETCC, std::swap(), llvm::ISD::VSELECT, and llvm::ISD::XOR.
Referenced by combineSelect().
|
static |
Definition at line 33706 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), i, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isNullConstant(), peekThroughBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::APInt::sextOrTrunc(), llvm::ISD::UNDEF, Vals, llvm::X86ISD::VZEXT, and llvm::APInt::zextOrTrunc().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 33315 of file X86ISelLowering.cpp.
References combineSetCCEFLAGS(), llvm::X86::COND_A, llvm::X86::COND_B, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), getSETCC(), llvm::SDNode::getSimpleValueType(), MaterializeSETB(), and llvm::X86ISD::SUB.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Combine an arbitrary chain of shuffles into a single instruction if possible.
This is the leaf of the recursive combine below. When we have found some chain of single-use x86 shuffle instructions and accumulated the combined shuffle mask represented by them, this will try to pattern match that mask into either a single instruction if there is a special purpose instruction for this operation, or into a PSHUFB instruction which is a fully general instruction but should only be used to replace chains over a certain depth.
Definition at line 26754 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::AND, llvm::any_of(), assert(), llvm::ArrayRef< T >::begin(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SmallVectorTemplateCommon< T >::data(), llvm::ArrayRef< T >::empty(), llvm::ArrayRef< T >::end(), llvm::X86ISD::FAND, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::APInt::getNullValue(), llvm::SDValue::getOpcode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSSE3(), llvm::X86Subtarget::hasVBMI(), llvm::X86Subtarget::hasVLX(), llvm::X86Subtarget::hasXOP(), i, llvm::MVT::i32, llvm::MVT::i8, is128BitLaneCrossingShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), isSequentialOrUndefInRange(), isSequentialOrUndefOrZeroInRange(), llvm::TargetLoweringBase::isTypeLegal(), isUndefOrZeroOrInRange(), llvm::BitmaskEnumDetail::Mask(), matchBinaryPermuteVectorShuffle(), matchBinaryVectorShuffle(), matchUnaryPermuteVectorShuffle(), matchUnaryVectorShuffle(), peekThroughBitcasts(), llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), scaleShuffleMask(), llvm::SmallVectorTemplateCommon< T >::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::MVT::v16f32, llvm::MVT::v16i16, llvm::MVT::v16i32, llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, llvm::MVT::v32i16, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i64, llvm::MVT::v64i8, llvm::MVT::v8f32, llvm::MVT::v8f64, llvm::MVT::v8i32, llvm::MVT::v8i64, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, and llvm::X86ISD::VZEXT_LOAD.
Referenced by combineX86ShufflesRecursively().
|
static |
Definition at line 27217 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), assert(), llvm::tgtok::Bits, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::count(), llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::APInt::getNullValue(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), i, llvm::MVT::isFloatingPoint(), llvm::SmallVectorTemplateCommon< T >::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineX86ShufflesRecursively().
|
static |
Fully generic combining of x86 shuffle instructions.
This should be the last combine run over the x86 shuffle instructions. Once they have been fully optimized, this will recursively consider all chains of single-use shuffle instructions, build a generic model of the cumulative shuffle operation, and check for simpler instructions which implement this operation. We use this primarily for two purposes:
1) Collapse generic shuffles to specialized single instructions when equivalent. In most cases, this is just an encoding size win, but sometimes we will collapse multiple generic shuffles into a single special-purpose shuffle. 2) Look for sequences of shuffle instructions with 3 or more total instructions, and replace them with the slightly more expensive SSSE3 PSHUFB instruction if available. We do this as the last combining step to ensure we avoid using PSHUFB if we can implement the shuffle with a suitable short sequence of other instructions. The PSHUFB will either use a register or have to read from memory and so is slightly (but only slightly) more expensive than the other shuffle instructions.
Because this is inherently a quadratic operation (for each shuffle in a chain, we recurse up the chain), the depth is limited to 8 instructions. This should never be an issue in practice as the shuffle lowering doesn't produce sequences of more than 8 instructions.
FIXME: We will currently miss some cases where the redundant shuffling would simplify under the threshold for PSHUFB formation because of combine-ordering. To fix this, we should do the redundant instruction combining in this recursive walk.
Definition at line 27330 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::any_of(), assert(), llvm::ArrayRef< T >::begin(), canonicalizeShuffleMaskWithCommute(), canWidenShuffleElements(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineX86ShuffleChain(), combineX86ShufflesConstants(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::SmallVectorBase::empty(), llvm::ArrayRef< T >::end(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), getZeroVector(), i, llvm::SDNode::isOnlyUserOf(), isTargetShuffleVariableMask(), llvm::MVT::isVector(), llvm::BitmaskEnumDetail::Mask(), peekThroughBitcasts(), peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SmallVectorImpl< T >::reserve(), resolveTargetShuffleInputs(), llvm::SmallVectorTemplateCommon< T >::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and std::swap().
Referenced by combineAnd(), combineShuffle(), and combineVectorShift().
|
static |
Definition at line 32605 of file X86ISelLowering.cpp.
References combineFneg(), combineIntegerAbs(), convertIntLogicToFPLogic(), foldVectorXorShiftIntoCmp(), foldXorTruncShiftIntoCmp(), llvm::X86Subtarget::hasCMov(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and isFNEG().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 33159 of file X86ISelLowering.cpp.
References llvm::ISD::AND, combineOrCmpEqZeroToCtlzSrl(), combineToExtendVectorInReg(), llvm::SelectionDAG::getConstant(), getDivRem8(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::is256BitVector(), llvm::isOneConstant(), promoteExtBeforeAdd(), llvm::X86ISD::SETCC_CARRY, llvm::ISD::TRUNCATE, and WidenMaskArithmetic().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Compute whether each element of a shuffle is zeroable.
A "zeroable" vector shuffle element is one which can be lowered to zero. Either it is an undef element in the shuffle mask, the element of the input referenced is undef, or the element of the input referenced is known to be zero. Many x86 shuffles can zero lanes cheaply and we often want to handle as many lanes with this technique as possible to simplify the remaining shuffle.
Definition at line 8025 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::APInt::getLoBits(), llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), i, llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), llvm::APInt::lshr(), peekThroughBitcasts(), llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerVectorShuffle().
|
static |
Concat two 128-bit vectors into a 256 bit vector using VINSERTF128 instructions.
This is used because creating CONCAT_VECTOR nodes of BUILD_VECTORS returns a larger BUILD_VECTOR while we're trying to lower large BUILD_VECTORS.
Definition at line 5082 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getUNDEF(), and insert128BitVector().
Referenced by getOnesVector(), and LowerAVXCONCAT_VECTORS().
|
static |
Definition at line 5089 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getUNDEF(), and insert256BitVector().
Referenced by LowerAVXCONCAT_VECTORS().
|
static |
Definition at line 8185 of file X86ISelLowering.cpp.
References i, and llvm::SmallBitVector::size().
Referenced by lowerVectorShuffleToEXPAND().
|
static |
Definition at line 6706 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::tgtok::In, llvm::ISD::isBuildVectorOfConstantSDNodes(), and llvm::SDValue::isUndef().
|
static |
If both input operands of a logic op are being cast from floating point types, try to convert this into a floating point logic node to avoid unnecessary moves from SSE to integer registers.
Definition at line 30667 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::ISD::DELETED_NODE, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i32, llvm::MVT::i64, llvm::EVT::isFloatingPoint(), llvm::ISD::OR, and llvm::ISD::XOR.
Referenced by combineAnd(), combineOr(), and combineXor().
|
static |
Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute.
The copy will be passed as a byval function parameter.
Definition at line 2604 of file X86ISelLowering.cpp.
References llvm::ISD::ArgFlagsTy::getByValAlign(), llvm::ISD::ArgFlagsTy::getByValSize(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMemcpy(), and llvm::MVT::i32.
|
static |
Definition at line 28557 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), llvm::MVT::i64, llvm::MVT::i8, and llvm::X86ISD::PSADBW.
Referenced by combineBasicSADPattern(), and combineLoopSADPattern().
|
static |
Generate unpacklo/unpackhi shuffle mask.
Definition at line 5118 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorBase::empty(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), i, and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().
Referenced by getUnpackh(), getUnpackl(), lowerVectorShuffleWithUNPCK(), and matchBinaryVectorShuffle().
|
static |
This function detects the AVG pattern between vectors of unsigned i8/i16, which is c = (a + b + 1) / 2, and replace this operation with the efficient X86ISD::AVG instruction.
Definition at line 31278 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::AVG, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::TargetLoweringBase::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSE2(), i, llvm::MVT::i8, llvm::BuildVectorSDNode::isConstant(), llvm::isPowerOf2_32(), llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::Max, llvm::Min, llvm::ISD::SRL, llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineStore(), and combineTruncate().
Definition at line 28507 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::i8, llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::SETCC, llvm::ISD::SETGT, llvm::ISD::SUB, and llvm::ISD::ZERO_EXTEND.
Referenced by combineBasicSADPattern(), and combineLoopSADPattern().
|
static |
Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.
Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a
Definition at line 6214 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), assert(), llvm::SmallBitVector::count(), llvm::count(), llvm::SmallBitVector::find_first(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MachineMemOperand::getFlags(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::EVT::getStoreSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasAnyUseOfValue(), i, llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::ISD::isNON_EXTLoad(), llvm::TargetLoweringBase::isOperationLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), llvm::ARM_MB::LD, llvm::ISD::LOAD, llvm::MachineMemOperand::MOVolatile, llvm::MVT::Other, peekThroughBitcasts(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ArrayRef< T >::size(), llvm::ISD::TokenFactor, llvm::SelectionDAG::UpdateNodeOperands(), and llvm::X86ISD::VZEXT_LOAD.
Referenced by combineShuffle(), and LowerINSERT_SUBVECTOR().
|
static |
Definition at line 15927 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::MVT::i1, llvm::MVT::isVector(), and llvm::X86ISD::KTEST.
|
static |
Emit Masked Truncating Store with signed or unsigned saturation.
Definition at line 19872 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getTargetMemSDNode(), llvm::SelectionDAG::getVTList(), llvm::BitmaskEnumDetail::Mask(), llvm::MVT::Other, and Ptr.
Referenced by LowerINTRINSIC_W_CHAIN().
|
static |
Definition at line 24391 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::addOperand(), llvm::X86::AddrNumOperands, llvm::BuildMI(), llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::X86Subtarget::getInstrInfo(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), i, and llvm::X86Subtarget::is64Bit().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 24319 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::addOperand(), llvm::BuildMI(), llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::MachineInstr::hasOneMemOperand(), i, llvm::MachineOperand::isImplicit(), llvm::MachineOperand::isReg(), llvm_unreachable, llvm::MachineInstr::memoperands_begin(), llvm::MachineInstr::memoperands_end(), and llvm::MachineInstr::setMemRefs().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 24283 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::addOperand(), llvm::BuildMI(), llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::MachineInstr::hasOneMemOperand(), i, llvm::MachineOperand::isImplicit(), llvm::MachineOperand::isReg(), llvm_unreachable, llvm::MachineInstr::memoperands_begin(), llvm::MachineInstr::memoperands_end(), and llvm::MachineInstr::setMemRefs().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 24374 of file X86ISelLowering.cpp.
References llvm::BuildMI(), llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::X86Subtarget::getInstrInfo(), llvm::MachineInstr::getOperand(), and llvm::MachineOperand::getReg().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
|
static |
Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0).
Definition at line 3185 of file X86ISelLowering.cpp.
References llvm::MachineFrameInfo::CreateFixedObject(), llvm::MachinePointerInfo::getFixedStack(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), llvm::SelectionDAG::getMachineFunction(), and llvm::SelectionDAG::getStore().
|
static |
Emit Truncating Store with signed or unsigned saturation.
Definition at line 19858 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getTargetMemSDNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::Other, and Ptr.
Referenced by LowerINTRINSIC_W_CHAIN().
|
static |
Definition at line 24353 of file X86ISelLowering.cpp.
References llvm::BuildMI(), llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineInstr::getDebugLoc(), llvm::X86Subtarget::getInstrInfo(), llvm::MachineInstr::getOperand(), and llvm::MachineOperand::getReg().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
|
static |
Utility function to emit xbegin specifying the start of an RTM region.
Definition at line 24226 of file X86ISelLowering.cpp.
References llvm::MachineBasicBlock::addLiveIn(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineBasicBlock::end(), llvm::MachineInstr::eraseFromParent(), llvm::MCInstrInfo::get(), llvm::MachineBasicBlock::getBasicBlock(), llvm::MachineInstr::getDebugLoc(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::insert(), MBB, llvm::MachineBasicBlock::splice(), and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
|
static |
Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.
This is a helper function of LowerToHorizontalOp(). This function expects two 256-bit vectors called V0 and V1. At first, each vector is split into two separate 128-bit vectors. Then, the resulting 128-bit vectors are used to implement two horizontal binary operations.
The kind of horizontal binary operation is defined by X86Opcode.
Mode specifies how the 128-bit parts of V0 and V1 are passed in input to the two new horizontal binop. When Mode is set, the first horizontal binop dag node would take as input the lower 128-bit of V0 and the upper 128-bit of V0. The second horizontal binop dag node would take as input the lower 128-bit of V1 and the upper 128-bit of V1. Example: HADD V0_LO, V0_HI HADD V1_LO, V1_HI
Otherwise, the first horizontal binop dag node takes as input the lower 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop dag node takes the upper 128-bit of V0 and the upper 128-bit of V1. Example: HADD V0_LO, V1_LO HADD V0_HI, V1_HI
If isUndefLO is set, then the algorithm propagates UNDEF to the lower 128-bits of the result. If isUndefHI is set, then UNDEF is propagated to the upper 128-bits of the result.
Definition at line 6928 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::AArch64CC::HI, llvm::MVT::is256BitVector(), llvm::SDNode::isUndef(), and llvm::AArch64CC::LO.
Referenced by LowerToHorizontalOp().
|
static |
Widen a vector input to a vector of NVT.
The input vector must have the same element type as NVT.
Definition at line 22908 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), i, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDValue::isUndef(), and llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back().
Referenced by LowerMGATHER(), LowerMLOAD(), LowerMSCATTER(), and LowerMSTORE().
|
static |
Generate a DAG to grab 128-bits from a vector > 128 bits.
This sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128 or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4 instructions or a simple subregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering EXTRACT_VECTOR_ELT operations easier.
Definition at line 4874 of file X86ISelLowering.cpp.
References assert(), extractSubVector(), llvm::SDValue::getValueType(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().
Referenced by combineStore(), ExpandHorizontalBinOp(), Lower256IntArith(), Lower256IntVSETCC(), LowerBITREVERSE(), LowerBITREVERSE_XOP(), LowerEXTRACT_SUBVECTOR(), LowerMUL_LOHI(), LowerVectorCTLZ(), and LowerVectorCTPOP().
|
static |
Generate a DAG to grab 256-bits from a 512-bit vector.
Definition at line 4882 of file X86ISelLowering.cpp.
References assert(), extractSubVector(), llvm::SDValue::getValueType(), and llvm::EVT::is512BitVector().
Referenced by Lower512IntArith(), LowerEXTRACT_SUBVECTOR(), and LowerVectorCTPOP().
|
static |
Definition at line 4839 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::isPowerOf2_32(), llvm::SDValue::isUndef(), llvm::makeArrayRef(), and llvm::SDNode::op_begin().
Referenced by extract128BitVector(), extract256BitVector(), LowerEXTEND_VECTOR_INREG(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), and truncateVectorCompareWithPACKSS().
|
static |
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1.
This should be called before type legalization because the pattern may not persist after that.
Definition at line 31234 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE42(), llvm::ISD::isBuildVectorAllOnes(), llvm::EVT::isSimple(), llvm::X86ISD::PCMPGT, llvm::MVT::SimpleTy, llvm::ISD::SRA, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8i16, and llvm::MVT::v8i32.
Referenced by combineXor().
|
static |
Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1)
Definition at line 31179 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::isOneConstant(), llvm::ISD::SETGT, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineXor().
|
static |
Definition at line 2737 of file X86ISelLowering.cpp.
References assert(), llvm::sys::path::begin(), llvm::sys::path::end(), llvm::X86Subtarget::is64Bit(), llvm::X86Subtarget::isCallingConvWin64(), and llvm::makeArrayRef().
|
static |
Definition at line 2755 of file X86ISelLowering.cpp.
References assert(), llvm::sys::path::begin(), llvm::sys::path::end(), llvm::MachineFunction::getFunction(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::is64Bit(), llvm::X86Subtarget::isCallingConvWin64(), llvm::makeArrayRef(), llvm::None, and llvm::X86Subtarget::useSoftFloat().
|
static |
Create a BT (Bit Test) node - Test bit BitNo in Src and set condition according to equal/not-equal condition code CC.
Definition at line 16387 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::X86ISD::BT, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::SelectionDAG::getNode(), getSETCC(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::SelectionDAG::MaskedValueIsZero(), llvm::ISD::SETEQ, and llvm::ISD::TRUNCATE.
Referenced by LowerAndToBT(), and LowerTruncateToBT().
|
static |
Definition at line 6379 of file X86ISelLowering.cpp.
References assert(), llvm::APInt::bitsToDouble(), llvm::APInt::bitsToFloat(), llvm::ConstantFP::get(), llvm::ConstantVector::get(), llvm::Type::getDoubleTy(), llvm::Type::getFloatTy(), llvm::Constant::getIntegerValue(), llvm::Type::getIntNTy(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::isFloatingPoint(), llvm::APInt::lshr(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), and llvm::APInt::trunc().
Referenced by LowerVectorBroadcast().
|
static |
Definition at line 4743 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), i, llvm::MVT::i32, llvm::MVT::i64, llvm::TargetLoweringBase::isTypeLegal(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), and Split().
Referenced by combineVSZext(), combineX86ShuffleChain(), combineX86ShufflesConstants(), lowerV8F32VectorShuffle(), lowerV8I32VectorShuffle(), and lowerVectorShuffleWithPERMV().
|
static |
Definition at line 4773 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::f32, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), i, llvm::MVT::i32, llvm::MVT::i64, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEsingle(), llvm::TargetLoweringBase::isTypeLegal(), llvm::APInt::lshr(), llvm::ArrayRef< T >::size(), llvm::SmallBitVector::size(), Split(), and llvm::APInt::trunc().
|
static |
(i8,i32 {s/z}ext ({s/u}divrem (i8 x, i8 y)) -> (i8,i32 ({s/u}divrem_sext_hreg (i8 x, i8 y) This exposes the {s/z}ext to the sdivrem lowering, so that it directly extends from AH (which we otherwise need to do contortions to access).
Definition at line 32929 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getResNo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::MVT::i8, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SDIVREM, llvm::X86ISD::SDIVREM8_SEXT_HREG, llvm::ISD::SIGN_EXTEND, llvm::ISD::UDIVREM, llvm::X86ISD::UDIVREM8_ZEXT_HREG, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSext(), and combineZext().
|
static |
Handles the lowering of builtin intrinsic that return the value of the extended control register.
Definition at line 19667 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_PAIR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::AArch64CC::HI, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::AArch64CC::LO, llvm::ISD::OR, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ISD::SHL, and llvm::XGETBV.
Referenced by LowerINTRINSIC_W_CHAIN(), and llvm::X86TargetLowering::ReplaceNodeResults().
Definition at line 4681 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), and llvm::MVT::getVectorElementType().
Referenced by llvm::X86::getExtractVEXTRACT128Immediate(), and llvm::X86::getExtractVEXTRACT256Immediate().
|
static |
Definition at line 5733 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::clear(), llvm::DecodeZeroExtendMask(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::MVT::getVectorNumElements(), i, llvm::BitmaskEnumDetail::Mask(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::SmallVectorTemplateCommon< T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLI, and llvm::X86ISD::VZEXT.
Referenced by resolveTargetShuffleInputs().
|
static |
Definition at line 19606 of file X86ISelLowering.cpp.
References getMaskNode(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i8, llvm::SDValue::isUndef(), and llvm::MVT::Other.
Referenced by LowerINTRINSIC_W_CHAIN().
Definition at line 4696 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), and llvm::MVT::getVectorElementType().
Referenced by llvm::X86::getInsertVINSERT128Immediate(), and llvm::X86::getInsertVINSERT256Immediate().
|
static |
Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics.
Definition at line 18603 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::MVT::bitsGT(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_ELEMENT, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::X86Subtarget::is32Bit(), llvm::isAllOnesConstant(), llvm::X86::isZeroNode(), llvm::MipsISD::Lo, llvm::ISD::TRUNCATE, llvm::MVT::v32i1, and llvm::MVT::v64i1.
Referenced by getGatherNode(), getPrefetchNode(), getScatterNode(), getVectorMaskingNode(), LowerINTRINSIC_W_CHAIN(), LowerINTRINSIC_WO_CHAIN(), and lowerVectorShuffleToEXPAND().
Helper for getByValTypeAlignment to determine the desired ByVal argument alignment.
Definition at line 1788 of file X86ISelLowering.cpp.
Referenced by llvm::X86TargetLowering::getByValTypeAlignment().
|
static |
Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width.
Definition at line 3204 of file X86ISelLowering.cpp.
References llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), i, llvm::BitmaskEnumDetail::Mask(), and llvm::SmallVectorTemplateBase< T, isPodLike >::push_back().
|
static |
If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value.
In the case of a vector, the returned constant will not contain undefined elements even if the input parameter does. This makes it suitable to be used as a replacement operand with operations (eg, bitwise-and) where an undef should not propagate.
Definition at line 32640 of file X86ISelLowering.cpp.
References llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), getZeroVector(), isNullFPScalarOrVectorConst(), and llvm::EVT::isVector().
Referenced by combineFAnd(), and combineFAndn().
|
static |
Returns a vector of specified type with all bits set.
Always build ones vectors as <4 x i32> or <8 x i32>. For 256-bit types with no AVX2 support, use two <4 x i32> inserted in a <8 x i32> appropriately. Then bitcast to their original type, ensuring they get CSE'd.
Definition at line 5100 of file X86ISelLowering.cpp.
References assert(), concat128BitVectors(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::MVT::i32, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::MVT::v4i32, and llvm::MVT::v8i32.
Referenced by combineVectorCompare(), lower1BitVectorShuffle(), LowerSIGN_EXTEND_AVX512(), and materializeVectorConstant().
|
static |
If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element.
Otherwise, return -1.
Definition at line 31447 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), i, llvm::MVT::i1, and llvm::SDValue::isUndef().
Referenced by getParamsForOneTrueMaskedElt().
|
static |
Given a masked memory load/store operation, return true if it has one mask bit set.
If it has one mask bit set, then also return the memory address of the scalar element to load/store, the vector index to insert/extract that scalar element, and the alignment for the scalar memory access.
Definition at line 31482 of file X86ISelLowering.cpp.
References llvm::MemSDNode::getAlignment(), llvm::MaskedLoadStoreSDNode::getBasePtr(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemoryVT(), getOneTrueElt(), llvm::EVT::getStoreSize(), llvm::EVT::getVectorElementType(), and llvm::MinAlign().
Referenced by reduceMaskedLoadToScalarLoad(), and reduceMaskedStoreToScalarStore().
|
static |
Definition at line 19647 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getMachineNode(), getMaskNode(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i8, and llvm::MVT::Other.
Referenced by LowerINTRINSIC_W_CHAIN().
|
static |
Get the PSHUF-style mask from PSHUF node.
This is a very minor wrapper around getTargetShuffleMask to easy forming v4 PSHUF-style masks that can be reused with such instructions.
Definition at line 27518 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetShuffleMask(), i, llvm_unreachable, llvm::BitmaskEnumDetail::Mask(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, and llvm::X86ISD::PSHUFLW.
Referenced by combineRedundantDWordShuffle(), combineRedundantHalfShuffle(), and combineTargetShuffle().
|
static |
Handles the lowering of builtin intrinsics that read performance monitor counters (x86_rdpmc).
Definition at line 19712 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_PAIR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::AArch64CC::HI, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::AArch64CC::LO, llvm::ISD::OR, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::X86ISD::RDPMC_DAG, and llvm::ISD::SHL.
Referenced by LowerINTRINSIC_W_CHAIN(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp).
This function is also used to custom lower READCYCLECOUNTER nodes.
Definition at line 19759 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_PAIR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::AArch64CC::HI, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::AArch64CC::LO, llvm::ISD::OR, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::X86ISD::RDTSCP_DAG, and llvm::ISD::SHL.
Referenced by LowerINTRINSIC_W_CHAIN(), LowerREADCYCLECOUNTER(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Creates an SDNode for a predicated scalar operation.
Mask, Op, PreservedSrc). The mask is coming as MVT::i8 and it should be truncated to MVT::i1 while lowering masking intrinsics. The main difference between ScalarMaskingNode and VectorMaskingNode is using "X86select" instead of "vselect". We just can't create the "vselect" node for a scalar instruction. Definition at line 18700 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::FSETCCM, llvm::X86ISD::FSETCCM_RND, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getSimpleValueType(), getZeroVector(), llvm::MVT::i1, llvm::isAllOnesConstant(), llvm::SDValue::isUndef(), llvm::ISD::OR, llvm::X86ISD::SELECTS, llvm::ISD::TRUNCATE, llvm::X86ISD::VFPCLASS, and llvm::X86ISD::VFPCLASSS.
Referenced by LowerINTRINSIC_WO_CHAIN().
|
static |
Try to get a scalar value for a specific element of a vector.
Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.
Definition at line 9278 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::isVector(), peekThroughBitcasts(), and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by lowerV2F64VectorShuffle(), and lowerVectorShuffleAsElementInsertion().
|
static |
Definition at line 19628 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getMachineNode(), getMaskNode(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i8, and llvm::MVT::Other.
Referenced by LowerINTRINSIC_W_CHAIN().
Definition at line 18724 of file X86ISelLowering.cpp.
References llvm::classifyEHPersonality(), llvm::Function::getPersonalityFn(), llvm::Function::hasPersonalityFn(), llvm::MSVC_CXX, llvm::MSVC_X86SEH, and llvm::report_fatal_error().
Referenced by recoverFramePointer().
|
static |
Helper for creating a X86ISD::SETCC node.
Definition at line 16379 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i8, and llvm::X86ISD::SETCC.
Referenced by combineCMov(), combineX86SetCC(), getBitTestCondition(), LowerCMP_SWAP(), LowerINTRINSIC_W_CHAIN(), LowerINTRINSIC_WO_CHAIN(), LowerXALUO(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Returns the scalar element that will make up the ith element of the result of the vector shuffle.
Definition at line 5848 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::isInteger(), isTargetShuffle(), llvm::EVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineShuffle().
|
static |
Return a vector_shuffle of the specified vector of zero or undef vector.
This produces a shuffle where the low element of V2 is swizzled into the zero/undef vector, landing at element Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
Definition at line 5153 of file X86ISelLowering.cpp.
References llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), and i.
Referenced by llvm::X86TargetLowering::BuildFILD().
|
static |
Definition at line 5201 of file X86ISelLowering.cpp.
References assert(), llvm::tgtok::Bits, llvm::SmallVectorBase::empty(), llvm::SmallBitVector::empty(), llvm::Constant::getAggregateElement(), llvm::APInt::getLowBitsSet(), llvm::APInt::getNullValue(), llvm::Type::getPrimitiveSizeInBits(), llvm::Type::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), getTargetConstantFromNode(), llvm::Value::getType(), llvm::Type::getVectorNumElements(), llvm::APInt::getZExtValue(), i, llvm::APInt::isAllOnesValue(), llvm::Type::isVectorTy(), llvm::APInt::lshr(), llvm::BitmaskEnumDetail::Mask(), peekThroughBitcasts(), llvm::SmallVectorImpl< T >::resize(), llvm::APInt::shl(), llvm::X86ISD::VBROADCAST, and llvm::APInt::zextOrTrunc().
Referenced by combineX86ShufflesConstants(), and getFauxShuffleMask().
Definition at line 5181 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SPII::Load, peekThroughBitcasts(), Ptr, llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.
Referenced by getTargetConstantBitsFromNode(), getTargetShuffleMask(), and isFNEG().
|
static |
Calculates the shuffle mask corresponding to the target-specific opcode.
If the mask could be calculated, returns it in Mask, returns the shuffle operands in Ops, and returns true. Sets IsUnary to true if only one source is used. Note that this will set IsUnary for shuffles which use a single input multiple times, and in those cases it will adjust the mask to only have indices within that single input. It is an error to call this with non-empty Mask/Ops vectors.
Definition at line 5393 of file X86ISelLowering.cpp.
References llvm::any_of(), assert(), llvm::X86ISD::BLENDI, llvm::DecodeBLENDMask(), llvm::DecodeINSERTPSMask(), llvm::DecodeMOVDDUPMask(), llvm::DecodeMOVHLPSMask(), llvm::DecodeMOVLHPSMask(), llvm::DecodeMOVSHDUPMask(), llvm::DecodeMOVSLDUPMask(), llvm::DecodePALIGNRMask(), llvm::DecodePSHUFBMask(), llvm::DecodePSHUFHWMask(), llvm::DecodePSHUFLWMask(), llvm::DecodePSHUFMask(), llvm::DecodePSLLDQMask(), llvm::DecodePSRLDQMask(), llvm::DecodeScalarMoveMask(), llvm::DecodeSHUFPMask(), llvm::DecodeUNPCKHMask(), llvm::DecodeUNPCKLMask(), llvm::DecodeVectorBroadcast(), llvm::DecodeVPERM2X128Mask(), llvm::DecodeVPERMIL2PMask(), llvm::DecodeVPERMILPMask(), llvm::DecodeVPERMMask(), llvm::DecodeVPERMV3Mask(), llvm::DecodeVPERMVMask(), llvm::DecodeVPPERMMask(), llvm::DecodeZeroMoveLowMask(), llvm::SmallVectorBase::empty(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), getTargetConstantFromNode(), getTargetShuffleMaskIndices(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::i8, llvm::X86ISD::INSERTPS, llvm_unreachable, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPD, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVLPD, llvm::X86ISD::MOVLPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::X86ISD::SHUFP, llvm::SM_SentinelZero, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMIV3, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VZEXT_MOVL.
Referenced by getPSHUFShuffleMask(), getShuffleScalarElt(), setTargetShuffleZeroElements(), and XFormVExtractWithShuffleIntoLoad().
|
static |
Definition at line 5303 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::ISD::BUILD_VECTOR, llvm::APInt::getLoBits(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZExtValue(), i, llvm::MVT::isVector(), llvm::X86::isZeroNode(), llvm::APInt::lshr(), llvm::SDNode::ops(), peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, Split(), llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VZEXT_MOVL.
Referenced by getTargetShuffleMask().
|
static |
Handle vector element shifts where the shift amount is a constant.
Takes immediate version of shift as input.
Definition at line 18467 of file X86ISelLowering.cpp.
References assert(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), i, llvm::MVT::i8, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDNode::isUndef(), llvm_unreachable, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.
Referenced by getTargetVShiftNode(), LowerMUL(), and LowerScalarImmediateShift().
|
static |
Handle vector element shifts where the shift amount may or may not be a constant.
Takes immediate version of shift as input.
Definition at line 18544 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetVShiftByConstNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm_unreachable, llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, llvm::X86ISD::VSRLI, llvm::X86ISD::VZEXT, and llvm::ISD::ZERO_EXTEND.
Referenced by combineVectorTruncationWithPACKSS(), LowerINTRINSIC_WO_CHAIN(), and LowerScalarVariableShift().
|
static |
Definition at line 14215 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getCopyFromReg(), llvm::MachineFunction::getFrameInfo(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::TLSModel::LocalDynamic, llvm::MVT::Other, llvm::MachineFrameInfo::setAdjustsStack(), llvm::MachineFrameInfo::setHasCalls(), llvm::X86ISD::TLSADDR, and llvm::X86ISD::TLSBASEADDR.
Referenced by LowerToTLSGeneralDynamicModel32(), LowerToTLSGeneralDynamicModel64(), and LowerToTLSLocalDynamicModel().
For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.
Modifies ExtractedFromVec to the real vector and returns the real index.
Definition at line 6604 of file X86ISelLowering.cpp.
References assert(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), and isUndefOrInRange().
Referenced by buildFromShuffleMostly().
|
static |
Returns a vector_shuffle node for an unpackh operation.
Definition at line 5142 of file X86ISelLowering.cpp.
References createUnpackShuffleMask(), llvm::SelectionDAG::getVectorShuffle(), and llvm::BitmaskEnumDetail::Mask().
Referenced by LowerAVXExtend().
|
static |
Returns a vector_shuffle node for an unpackl operation.
Definition at line 5134 of file X86ISelLowering.cpp.
References createUnpackShuffleMask(), llvm::SelectionDAG::getVectorShuffle(), and llvm::BitmaskEnumDetail::Mask().
Referenced by llvm::X86TargetLowering::BuildFILD(), and LowerAVXExtend().
Get a 4-lane 8-bit shuffle immediate for a mask.
This helper function produces an 8-bit shuffle immediate corresponding to the ubiquitous shuffle encoding scheme used in x86 instructions for shuffling 4 lanes. It can be used with most of the PSHUF instructions for example.
NB: We rely heavily on "undef" masks preserving the input lane.
Definition at line 7997 of file X86ISelLowering.cpp.
References assert(), and llvm::ArrayRef< T >::size().
Referenced by getV4X86ShuffleImm8ForMask(), matchBinaryPermuteVectorShuffle(), and matchUnaryPermuteVectorShuffle().
|
static |
Definition at line 8012 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), getV4X86ShuffleImm(), and llvm::MVT::i8.
Referenced by combineRedundantDWordShuffle(), combineRedundantHalfShuffle(), combineTargetShuffle(), lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerV8I32VectorShuffle(), lowerV8I64VectorShuffle(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), and lowerVectorShuffleWithSHUFPS().
|
static |
Reads two 32 bit registers and creates a 64 bit mask value.
| VA | The current 32 bit value that need to be assigned. | |
| NextVA | The next 32 bit value that need to be assigned. | |
| Root | The parent DAG node. | |
| [in,out] | InFlag | Represents SDvalue in the parent DAG node for glue purposes. In the case the DAG is already using physical register instead of virtual, we should glue our new SDValue to InFlag SDvalue. |
Definition at line 2402 of file X86ISelLowering.cpp.
References llvm::MachineFunction::addLiveIn(), assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getCopyFromReg(), llvm::CCValAssign::getLocReg(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::CCValAssign::getValVT(), llvm::X86Subtarget::hasBWI(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::X86Subtarget::is32Bit(), llvm::CCValAssign::isRegLoc(), llvm::MipsISD::Lo, llvm::MVT::v32i1, and llvm::MVT::v64i1.
|
static |
Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics.
Definition at line 18654 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::CMPM, llvm::X86ISD::CMPMU, llvm::X86ISD::CVTPS2PH, getMaskNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::i1, llvm::isAllOnesConstant(), llvm::SDValue::isUndef(), llvm::ISD::OR, llvm::X86ISD::PCMPEQM, llvm::X86ISD::PCMPGTM, llvm::X86ISD::SELECT, llvm::X86ISD::VFPCLASS, llvm::X86ISD::VFPCLASSS, llvm::ISD::VSELECT, llvm::X86ISD::VTRUNC, llvm::X86ISD::VTRUNCS, and llvm::X86ISD::VTRUNCUS.
Referenced by LowerINTRINSIC_WO_CHAIN().
|
static |
Return a vector logical shift node.
Definition at line 6123 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getScalarShiftAmountTy(), llvm::EVT::is128BitVector(), llvm::MVT::v16i8, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.
|
static |
Returns a vector of specified type with all zero elements.
Definition at line 4814 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), and llvm::MVT::v4f32.
Referenced by combineVectorCompare(), combineVectorShift(), combineX86ShuffleChain(), combineX86ShufflesRecursively(), getGatherNode(), getNullFPConstForNullVal(), getScalarMaskingNode(), getShuffleVectorZeroOrUndef(), getVectorMaskingNode(), insert1BitVector(), lower1BitVectorShuffle(), LowerAVXExtend(), LowerBuildVectorv16i8(), LowerBuildVectorv4x32(), LowerBuildVectorv8i16(), LowerCONCAT_VECTORSvXi1(), LowerHorizontalByteSum(), LowerINTRINSIC_WO_CHAIN(), LowerMUL(), LowerScalarImmediateShift(), LowerShift(), LowerSIGN_EXTEND_AVX512(), lowerV16I8VectorShuffle(), LowerVectorCTLZInRegLUT(), lowerVectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), lowerVectorShuffleToEXPAND(), LowerVSETCC(), matchBinaryPermuteVectorShuffle(), materializeVectorConstant(), and performShiftToAllZeros().
Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
Definition at line 4323 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, and llvm::X86::COND_P.
Referenced by combineCMov().
return true if Op has a use that doesn't just read flags.
Definition at line 15908 of file X86ISelLowering.cpp.
References llvm::ISD::BRCOND, llvm::SDNode::getOpcode(), llvm::SDNode::use_iterator::getOperandNo(), llvm::SDNode::hasOneUse(), llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), and llvm::SDNode::use_end().
|
static |
Generate a DAG to put 128-bits into a vector > 128 bits.
This sets things up to match to an AVX VINSERTF128/VINSERTI128 or AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a simple superregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering INSERT_VECTOR_ELT operations easier.
Definition at line 4918 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BLENDI, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, insertSubVector(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::MVT::isFloatingPoint(), llvm::SDValue::isUndef(), llvm::RegState::Undef, llvm::MVT::v8f32, and llvm::MVT::v8i32.
Referenced by combineLoad(), concat128BitVectors(), LowerINSERT_SUBVECTOR(), LowerSCALAR_TO_VECTOR(), and lowerV4X128VectorShuffle().
|
static |
Insert i1-subvector to i1-vector.
Definition at line 4975 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasDQI(), i, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), llvm::BitmaskEnumDetail::Mask(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::RegState::Undef, llvm::MVT::v16i1, llvm::MVT::v8i1, llvm::X86ISD::VSHLI, and llvm::X86ISD::VSRLI.
Referenced by LowerINSERT_SUBVECTOR().
|
static |
Definition at line 4968 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getValueType(), insertSubVector(), and llvm::EVT::is256BitVector().
Referenced by concat256BitVectors(), and LowerINSERT_SUBVECTOR().
|
static |
Definition at line 4888 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_SUBVECTOR, llvm::isPowerOf2_32(), and llvm::SDValue::isUndef().
Referenced by insert128BitVector(), and insert256BitVector().
Test whether there are elements crossing 128-bit lanes in this shuffle mask.
X86 divides up its shuffles into in-lane and cross-lane shuffle operations and we routinely test for these.
Definition at line 7829 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), i, and llvm::ArrayRef< T >::size().
Referenced by combineX86ShuffleChain(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerVectorShuffleByMerging128BitLanes(), and matchUnaryPermuteVectorShuffle().
|
static |
Test whether a shuffle mask is equivalent within each 128-bit lane.
Definition at line 7879 of file X86ISelLowering.cpp.
References isRepeatedShuffleMask().
Referenced by lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV16F32VectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV32I16VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I32VectorShuffle(), lowerV8I64VectorShuffle(), lowerVectorShuffleAsBlend(), matchUnaryPermuteVectorShuffle(), and matchVectorShuffleAsByteRotate().
|
static |
Test whether a shuffle mask is equivalent within each 256-bit lane.
Definition at line 7886 of file X86ISelLowering.cpp.
References isRepeatedShuffleMask().
Referenced by lowerV8F64VectorShuffle(), lowerV8I64VectorShuffle(), and matchUnaryPermuteVectorShuffle().
|
static |
Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB operation.
If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1 operation are written to the parameters Opnd0 and Opnd1.
Definition at line 6968 of file X86ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE3(), i, llvm::SDValue::isUndef(), std::swap(), llvm::ISD::UNDEF, llvm::MVT::v16f32, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v8f32, and llvm::MVT::v8f64.
Referenced by combineShuffleToAddSubOrFMAddSub(), and lowerToAddSubOrFMAddSub().
|
static |
Returns true iff the shuffle node N can be replaced with ADDSUB operation.
If true is returned then the operands of ADDSUB operation are written to the parameters Opnd0 and Opnd1.
We combine shuffle to ADDSUB directly on the abstract vector shuffle nodes so it is easier to generically match. We also insert dummy vector shuffle nodes for the operands which explicitly discard the lanes which are unused by this operation to try to flow through the rest of the combiner the fact that they're unused.
Definition at line 28046 of file X86ISelLowering.cpp.
References llvm::ArrayRef< T >::begin(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::ArrayRef< T >::end(), llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE3(), isShuffleEquivalent(), llvm::BitmaskEnumDetail::Mask(), N, std::swap(), llvm::MVT::v16f32, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v8f32, llvm::MVT::v8f64, and llvm::ISD::VECTOR_SHUFFLE.
Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR.
Definition at line 17945 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::ISD::OR, and llvm::X86ISD::SETCC.
Referenced by combineCompareEqual().
|
static |
Definition at line 24783 of file X86ISelLowering.cpp.
References llvm::MachineInstr::getOpcode().
|
static |
Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB(Opnd0, Opnd1) into FMADDSUB(x, y, Opnd1).
If (and only if) true is returned, the operands of FMADDSUB are written to parameters Opnd0, Opnd1, Opnd2.
Prior to calling this function it should be known that there is some SDNode that potentially can be replaced with an X86ISD::ADDSUB operation using Opnd0 and Opnd1 as operands. Also, this method is called before replacement of such SDNode with ADDSUB operation. Thus the number of Opnd0 uses is expected to be equal to 2. For example, this function may be called for the following IR: AB = fmul fast <2 x double> A, B Sub = fsub fast <2 x double> AB, C Add = fadd fast <2 x double> AB, C Addsub = shufflevector <2 x double> Sub, <2 x double> Add, <2 x i32> <i32 0, i32 3> There is a def for Addsub here, which potentially can be replaced by X86ISD::ADDSUB operation: Addsub = X86ISD::ADDSUB AB, C and such ADDSUB can further be replaced with FMADDSUB: Addsub = FMADDSUB A, B, C.
The main reason why this method is called before the replacement of the recognized ADDSUB idiom with ADDSUB operation is that such replacement is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit FMADDSUB is.
Definition at line 7095 of file X86ISelLowering.cpp.
References llvm::TargetOptions::AllowFPOpFusion, llvm::FPOpFusion::Fast, llvm::ISD::FMUL, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTarget(), llvm::X86Subtarget::hasAnyFMA(), llvm::TargetMachine::Options, llvm::TargetOptions::UnsafeFPMath, and llvm::SDNode::use_size().
Referenced by combineShuffleToAddSubOrFMAddSub(), and lowerToAddSubOrFMAddSub().
Returns the negated value if the node N flips sign of FP value.
FP-negation node may have different forms: FNEG(x) or FXOR (x, 0x80000000). AVX512F does not have FXOR, so FNEG is lowered as (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))). In this case we go though all bitcasts.
Definition at line 32482 of file X86ISelLowering.cpp.
References C, llvm::ISD::FNEG, llvm::X86ISD::FXOR, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::APInt::getSignBit(), getTargetConstantFromNode(), llvm::SDValue::getValueType(), llvm::EVT::isFloatingPoint(), peekThroughBitcasts(), llvm::X86ISD::VBROADCAST, and llvm::ISD::XOR.
Referenced by combineFMA(), combineFneg(), combineFOr(), and combineXor().
|
static |
Check if RC is a vector register class.
I.e., FR* / VR* or one of their variant.
Definition at line 34520 of file X86ISelLowering.cpp.
References llvm::TargetRegisterClass::hasSuperClassEq().
Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().
|
static |
Check if RC is a general purpose register class.
I.e., GR* or one of their variant.
Definition at line 34510 of file X86ISelLowering.cpp.
References llvm::TargetRegisterClass::hasSuperClassEq().
Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().
|
static |
Return true if N implements a horizontal binop and return the operands for the horizontal binop into V0 and V1.
This is a helper function of LowerToHorizontalOp(). This function checks that the build_vector N in input implements a horizontal operation. Parameter Opcode defines the kind of horizontal operation to match. For example, if Opcode is equal to ISD::ADD, then this function checks if N implements a horizontal arithmetic add; if instead Opcode is equal to ISD::SUB, then this function checks if this is a horizontal arithmetic sub.
This function only analyzes elements of N whose indices are in range [BaseIdx, LastIdx).
Definition at line 6814 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), i, llvm::SDValue::isUndef(), llvm::SDNode::isUndef(), and llvm::EVT::isVector().
Referenced by combineAdd(), combineFaddFsub(), combineSub(), and LowerToHorizontalOp().
Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS.
A horizontal operation performs the binary operation on successive elements of its first operand, then on successive elements of its second operand, returning the resulting values in a vector. For example, if A = < float a0, float a1, float a2, float a3 > and B = < float b0, float b1, float b2, float b3 > then the result of doing a horizontal operation on A and B is A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >. In short, LHS and RHS are inspected to see if LHS op RHS is of the form A horizontal-op B, for some already available A and B, and if so then LHS is set to A, RHS to B, and the routine returns 'true'. Note that the binary operation should have the property that if one of the operands is UNDEF then the result is UNDEF.
Definition at line 32057 of file X86ISelLowering.cpp.
References A, assert(), B, llvm::SmallVectorTemplateCommon< T >::begin(), llvm::ArrayRef< T >::begin(), C, llvm::ShuffleVectorSDNode::commuteMask(), D, llvm::ArrayRef< T >::end(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), i, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::SDValue::isUndef(), and llvm::ISD::VECTOR_SHUFFLE.
|
static |
Definition at line 8099 of file X86ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), i, and llvm::SmallBitVector::size().
Referenced by lowerVectorShuffleToEXPAND().
Tiny helper function to identify a no-op mask.
This is a somewhat boring predicate function. It checks whether the mask array input, which is assumed to be a single-input shuffle mask of the kind used by the X86 shuffle instructions (not a fully general ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an in-place shuffle are 'no-op's.
Definition at line 7815 of file X86ISelLowering.cpp.
References assert(), i, and llvm::ArrayRef< T >::size().
Referenced by combineTargetShuffle(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), and lowerVectorShuffleAsPermuteAndUnpack().
Definition at line 32630 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::ISD::isBuildVectorAllZeros(), and llvm::isNullFPConstant().
Referenced by combineFAndn(), combineFOr(), and getNullFPConstForNullVal().
|
static |
Test whether a shuffle mask is equivalent within each sub-lane.
This checks a shuffle mask to see if it is performing the same lane-relative shuffle in each sub-lane. This trivially implies that it is also not lane-crossing. It may however involve a blend from the same lane of a second vector.
The specific repeated shuffle mask is populated in RepeatedMask, as it is non-trivial to compute in the face of undef lanes. The representation is suitable for use with existing 128-bit shuffles as entries from the second vector have been remapped to [LaneSize, 2*LaneSize).
Definition at line 7849 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::MVT::getScalarSizeInBits(), i, llvm::ArrayRef< T >::size(), and llvm::SM_SentinelUndef.
Referenced by is128BitLaneRepeatedShuffleMask(), and is256BitLaneRepeatedShuffleMask().
|
static |
Test whether a target shuffle mask is equivalent within each sub-lane.
Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.
Definition at line 7893 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::MVT::getScalarSizeInBits(), i, isUndefOrZero(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by matchBinaryPermuteVectorShuffle().
|
static |
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size].
or is undef.
Definition at line 4516 of file X86ISelLowering.cpp.
References i, and isUndefOrEqual().
Referenced by combineX86ShuffleChain(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), lowerVectorShuffleWithUndefHalf(), matchUnaryPermuteVectorShuffle(), and matchVectorShuffleAsShift().
|
static |
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero.
Definition at line 4527 of file X86ISelLowering.cpp.
References i, and isUndefOrZero().
Referenced by combineX86ShuffleChain(), and matchBinaryPermuteVectorShuffle().
|
static |
Checks whether a shuffle mask is equivalent to an explicit list of arguments.
This is a fast way to test a shuffle mask against a fixed pattern:
if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }
It returns true if the mask is exactly as wide as the argument list, and each element of the mask is either -1 (signifying undef) or the value given in the argument.
Definition at line 7937 of file X86ISelLowering.cpp.
References assert(), llvm::dyn_cast(), i, llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by combineTargetShuffle(), isAddSub(), lowerV16F32VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2X128VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4X128VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerVectorShuffleAsBroadcast(), and lowerVectorShuffleWithUNPCK().
Helper to test for a load that can be folded with x86 shuffles.
This is particularly important because the set of instructions varies significantly based on whether the operand is a load or not.
Definition at line 9306 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::ISD::isNON_EXTLoad(), and peekThroughBitcasts().
Referenced by combineTargetShuffle(), lowerV2F64VectorShuffle(), and lowerVectorShuffleAsBroadcast().
Test whether the specified input (0 or 1) is in-place blended by the given mask.
This returns true if the elements from a particular input are already in the slot required by the given mask and require no permutation.
Definition at line 11910 of file X86ISelLowering.cpp.
References assert(), i, and llvm::ArrayRef< T >::size().
Referenced by lowerV4F64VectorShuffle(), and lowerV4I64VectorShuffle().
Test whether this can be lowered with a single SHUFPS instruction.
This is used to disable more specialized lowerings when the shufps lowering will happen to be efficient.
Definition at line 10070 of file X86ISelLowering.cpp.
References assert(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I32VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4I32VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Definition at line 2784 of file X86ISelLowering.cpp.
References A, B, llvm::SmallVectorTemplateCommon< T >::begin(), llvm::SmallVectorTemplateCommon< T >::end(), and llvm::CCValAssign::getValNo().
Definition at line 4081 of file X86ISelLowering.cpp.
References llvm::X86ISD::BLENDI, llvm::X86ISD::INSERTPS, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPD, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVLPD, llvm::X86ISD::MOVLPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMIV3, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VZEXT_MOVL.
Referenced by combineShuffle(), getShuffleScalarElt(), isUseOfShuffle(), setTargetShuffleZeroElements(), and XFormVExtractWithShuffleIntoLoad().
Checks whether a target shuffle mask is equivalent to an explicit pattern.
The masks must be exactly the same width.
If an element in Mask matches SM_SentinelUndef (-1) then the corresponding value in ExpectedMask is always accepted. Otherwise the indices must match.
SM_SentinelZero is accepted as a valid negative index but must match in both.
Definition at line 7972 of file X86ISelLowering.cpp.
References i, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by matchBinaryVectorShuffle(), and matchUnaryVectorShuffle().
Definition at line 4121 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::PSHUFB, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMIV3, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, and llvm::X86ISD::VPPERM.
Referenced by combineX86ShufflesRecursively().
|
static |
Definition at line 17111 of file X86ISelLowering.cpp.
References llvm::tgtok::Bits, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SelectionDAG::MaskedValueIsZero(), and llvm::ISD::TRUNCATE.
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value.
Definition at line 4475 of file X86ISelLowering.cpp.
References i, and llvm::SM_SentinelUndef.
Referenced by lowerVectorShuffleAsSpecificZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), lowerVectorShuffleWithUndefHalf(), and matchBinaryPermuteVectorShuffle().
|
static |
Val is the undef sentinel value or equal to the specified value.
Definition at line 4464 of file X86ISelLowering.cpp.
References llvm::SM_SentinelUndef.
Referenced by isSequentialOrUndefInRange(), and matchUnaryVectorShuffle().
|
static |
Return true if Val is undef or if its value falls within the specified range (L, H].
Definition at line 4484 of file X86ISelLowering.cpp.
References llvm::MipsISD::Hi, and llvm::SM_SentinelUndef.
Referenced by getUnderlyingExtractedFromVec(), isUndefOrInRange(), matchBinaryPermuteVectorShuffle(), and matchUnaryPermuteVectorShuffle().
Return true if every element in Mask is undef or if its value falls within the specified range (L, H].
Definition at line 4490 of file X86ISelLowering.cpp.
References isUndefOrInRange().
|
static |
Val is either the undef or zero sentinel value.
Definition at line 4469 of file X86ISelLowering.cpp.
References llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineTargetShuffle(), isRepeatedTargetShuffleMask(), isSequentialOrUndefOrZeroInRange(), isUndefOrZeroInRange(), isUndefOrZeroOrInRange(), and matchUnaryPermuteVectorShuffle().
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero.
Definition at line 4537 of file X86ISelLowering.cpp.
References i, and isUndefOrZero().
Referenced by matchBinaryPermuteVectorShuffle(), and matchUnaryVectorShuffle().
|
static |
Return true if Val is undef, zero or if its value falls within the specified range (L, H].
Definition at line 4500 of file X86ISelLowering.cpp.
References llvm::MipsISD::Hi, and isUndefOrZero().
Referenced by combineX86ShuffleChain(), and isUndefOrZeroOrInRange().
Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H].
Definition at line 4506 of file X86ISelLowering.cpp.
References isUndefOrZeroOrInRange().
Definition at line 6402 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, isTargetShuffle(), and llvm::SDNode::uses().
Referenced by LowerVectorBroadcast().
Return true if the specified EXTRACT_SUBVECTOR operand specifies a vector extract that is suitable for instruction that extract 128 or 256 bit vectors.
Definition at line 4631 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), and llvm::SDNode::getSimpleValueType().
Referenced by llvm::X86::isVEXTRACT128Index(), and llvm::X86::isVEXTRACT256Index().
Return true if the specified INSERT_SUBVECTOR operand specifies a subvector insert that is suitable for input to insertion of 128 or 256-bit subvectors.
Definition at line 4650 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), and llvm::SDNode::getSimpleValueType().
Referenced by llvm::X86::isVINSERT128Index(), and llvm::X86::isVINSERT256Index().
Return true if the condition is an unsigned comparison operation.
Definition at line 4208 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, and llvm_unreachable.
Return true if opcode is a X86 logical comparison.
Definition at line 17093 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::X86ISD::ADD, llvm::X86ISD::AND, llvm::X86ISD::CMP, llvm::X86ISD::COMI, llvm::X86ISD::DEC, llvm::SDValue::getOpcode(), llvm::SDValue::getResNo(), llvm::X86ISD::INC, llvm::X86ISD::OR, llvm::X86ISD::SAHF, llvm::X86ISD::SBB, llvm::X86ISD::SMUL, llvm::X86ISD::SUB, llvm::X86ISD::UCOMI, llvm::X86ISD::UMUL, and llvm::X86ISD::XOR.
Return true if node is an ISD::XOR of a X86ISD::SETCC and 1 and that the SETCC node has a single use.
Definition at line 17957 of file X86ISelLowering.cpp.
References llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isOneConstant(), llvm::X86ISD::SETCC, and llvm::ISD::XOR.
|
static |
Dispatching routine to lower various 128-bit x86 vector shuffles.
This routine breaks down the specific type of 128-bit shuffle and dispatches to the lowering routines accordingly.
Definition at line 11366 of file X86ISelLowering.cpp.
References llvm_unreachable, lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4I32VectorShuffle(), lowerV8I16VectorShuffle(), llvm::MVT::SimpleTy, llvm::MVT::v16i8, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4i32, and llvm::MVT::v8i16.
Referenced by lowerVectorShuffle().
|
static |
Definition at line 13221 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::CVT2MASK, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), llvm_unreachable, llvm::ISD::SIGN_EXTEND, llvm::MVT::SimpleTy, llvm::ISD::TRUNCATE, llvm::MVT::v16i1, llvm::MVT::v16i32, llvm::MVT::v2i1, llvm::MVT::v2i64, llvm::MVT::v32i1, llvm::MVT::v32i16, llvm::MVT::v4i1, llvm::MVT::v4i32, llvm::MVT::v64i1, llvm::MVT::v64i8, llvm::MVT::v8i1, and llvm::MVT::v8i64.
Referenced by lowerVectorShuffle().
|
static |
High-level routine to lower various 256-bit x86 vector shuffles.
This routine either breaks down the specific type of a 256-bit x86 vector shuffle or splits it into two 128-bit shuffles and fuses the results back together based on the available instructions.
Definition at line 12693 of file X86ISelLowering.cpp.
References llvm::count_if(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::isInteger(), llvm_unreachable, lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I32VectorShuffle(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleWithUndefHalf(), llvm::MVT::SimpleTy, splitAndLowerVectorShuffle(), llvm::MVT::v16i16, llvm::MVT::v32i8, llvm::MVT::v4f64, llvm::MVT::v4i64, llvm::MVT::v8f32, and llvm::MVT::v8i32.
Referenced by lowerVectorShuffle().
|
static |
Break a 256-bit integer operation into two new 128-bit ones and then concatenate the result back.
Definition at line 20749 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is256BitVector(), and llvm::MVT::isInteger().
Referenced by LowerADD(), LowerMINMAX(), LowerMUL(), LowerMULH(), LowerRotate(), and LowerSUB().
|
static |
Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back.
Definition at line 16539 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is256BitVector(), and llvm::ISD::SETCC.
Referenced by LowerVSETCC().
|
static |
High-level routine to lower various 512-bit x86 vector shuffles.
This routine either breaks down the specific type of a 512-bit x86 vector shuffle or splits it into two 256-bit shuffles and fuses the results back together based on the available instructions.
Definition at line 13171 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), llvm::X86Subtarget::hasAVX512(), llvm_unreachable, lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV32I16VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I64VectorShuffle(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsElementInsertion(), llvm::MVT::SimpleTy, llvm::ArrayRef< T >::size(), llvm::MVT::v16f32, llvm::MVT::v16i32, llvm::MVT::v32i16, llvm::MVT::v64i8, llvm::MVT::v8f64, and llvm::MVT::v8i64.
Referenced by lowerVectorShuffle().
|
static |
Break a 512-bit integer operation into two new 256-bit ones and then concatenate the result back.
Definition at line 20778 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extract256BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is512BitVector(), and llvm::MVT::isInteger().
Referenced by LowerMUL().
|
static |
Definition at line 20805 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), llvm::MVT::i1, llvm::MVT::is256BitVector(), llvm::MVT::isInteger(), Lower256IntArith(), and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 22827 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::X86ISD::ADD, llvm::ISD::ADDC, llvm::ISD::ADDE, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, llvm::X86ISD::SBB, llvm::X86ISD::SUB, llvm::ISD::SUBC, and llvm::ISD::SUBE.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 20287 of file X86ISelLowering.cpp.
References llvm::SDValue::getOperand().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Result of 'and' is compared against zero. Change to a BT node if possible.
Definition at line 16416 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::ISD::Constant, llvm::APInt::countLeadingOnes(), getBitTestCondition(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::isOneConstant(), llvm::isPowerOf2_64(), llvm::isUInt< 32 >(), llvm::Log2_64_Ceil(), llvm::ISD::SHL, llvm::ISD::SRL, std::swap(), and llvm::ISD::TRUNCATE.
|
static |
Definition at line 15322 of file X86ISelLowering.cpp.
References llvm::X86Subtarget::hasFp256(), and LowerAVXExtend().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 6136 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::MVT::f32, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFrameInfo(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::i32, llvm::SelectionDAG::InferPtrAlignment(), llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::MachineFrameInfo::isFixedObjectIndex(), llvm::ISD::isNormalLoad(), llvm::ARM_MB::LD, llvm::BitmaskEnumDetail::Mask(), Ptr, and llvm::MachineFrameInfo::setObjectAlignment().
|
static |
Definition at line 22252 of file X86ISelLowering.cpp.
References llvm::CrossThread, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::hasMFence(), llvm::MVT::i32, llvm::MVT::i8, llvm::X86ISD::MEMBARRIER, llvm::X86ISD::MFENCE, llvm::MVT::Other, llvm::SequentiallyConsistent, and Zero.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 22803 of file X86ISelLowering.cpp.
References llvm::ISD::ATOMIC_SWAP, llvm::SelectionDAG::getAtomic(), llvm::SDValue::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::SequentiallyConsistent.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Lower atomic_load_ops into LOCK-prefixed operations.
Definition at line 22770 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_SUB, llvm::SelectionDAG::getAtomic(), llvm::SelectionDAG::getConstant(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SDValue::getValue(), llvm::SDNode::hasAnyUseOfValue(), lowerAtomicArithWithLOCK(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and llvm::ISD::SUB.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 22740 of file X86ISelLowering.cpp.
References llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::ATOMIC_LOAD_OR, llvm::ISD::ATOMIC_LOAD_SUB, llvm::ISD::ATOMIC_LOAD_XOR, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::X86ISD::LADD, llvm::X86ISD::LAND, llvm_unreachable, llvm::X86ISD::LOR, llvm::X86ISD::LSUB, llvm::X86ISD::LXOR, N, and llvm::MVT::Other.
Referenced by combineSetCCAtomicArith(), and lowerAtomicArith().
|
static |
Definition at line 7676 of file X86ISelLowering.cpp.
References assert(), concat128BitVectors(), concat256BitVectors(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::NVPTX::PTXLdStInstCode::V2, and llvm::NVPTX::PTXLdStInstCode::V4.
Referenced by LowerCONCAT_VECTORS().
|
static |
Definition at line 15245 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasInt256(), llvm::MVT::i1, llvm::MVT::is512BitVector(), llvm::RegState::Undef, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8i16, llvm::MVT::v8i32, llvm::X86ISD::VZEXT, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerANY_EXTEND(), and LowerZERO_EXTEND().
|
static |
Definition at line 22326 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::append(), assert(), llvm::ISD::EXTRACT_ELEMENT, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasMMX(), llvm::X86Subtarget::hasSSE2(), i, llvm::MVT::i32, llvm::MVT::i64, llvm::X86Subtarget::is64Bit(), llvm::MVT::isVector(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::MVT::v2f64, llvm::MVT::v2i32, llvm::MVT::v4i16, and llvm::MVT::v8i8.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 22684 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITREVERSE, llvm::ISD::CONCAT_VECTORS, extract128BitVector(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::X86Subtarget::hasXOP(), i, llvm::MVT::i8, LowerBITREVERSE_XOP(), llvm::ISD::OR, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), and llvm::ISD::SRL.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 22631 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITREVERSE, llvm::ISD::CONCAT_VECTORS, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), i, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::isVector(), llvm::BitmaskEnumDetail::Mask(), llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v16i8, and llvm::X86ISD::VPPERM.
Referenced by LowerBITREVERSE().
|
static |
Definition at line 16567 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::i1, llvm_unreachable, llvm::ISD::OR, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, and llvm::ISD::XOR.
Referenced by LowerVSETCC().
|
static |
If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors.
NOTE: Its not in our interest to start make a general purpose vectorizer from this, but enough scalar bit operations are created from the later legalization + scalarization stages to need basic support.
Definition at line 7281 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), i, llvm::TargetLoweringBase::isOperationLegalOrPromote(), llvm::SDNode::ops(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), and llvm::ISD::XOR.
|
static |
Custom lower build_vector of v16i8.
Definition at line 5916 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), i, llvm::MVT::i16, llvm::MVT::i8, llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::OR, llvm::ISD::SHL, llvm::MVT::v16i8, llvm::MVT::v8i16, and llvm::ISD::ZERO_EXTEND.
|
static |
Custom lower build_vector of v4i32 or v4f32.
Definition at line 6017 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), i, llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), llvm::NVPTX::PTXLdStInstCode::V2, and llvm::MVT::v4f32.
|
static |
Custom lower build_vector of v8i16.
Definition at line 5986 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), getZeroVector(), i, llvm::ISD::INSERT_VECTOR_ELT, and llvm::MVT::v8i16.
|
static |
Definition at line 22286 of file X86ISelLowering.cpp.
References assert(), llvm::X86::COND_E, llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getOperand(), getSETCC(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::Glue, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86Subtarget::is64Bit(), llvm::X86ISD::LCMPXCHG_DAG, llvm_unreachable, llvm::MVT::Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::MVT::SimpleTy, Success, and T.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 7779 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNumOperands(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::i1, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerAVXCONCAT_VECTORS(), and LowerCONCAT_VECTORSvXi1().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 7702 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MipsISD::Hi, i, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::isPowerOf2_32(), llvm::SDValue::isUndef(), llvm::MipsISD::Lo, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::RegState::Undef, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by LowerCONCAT_VECTORS().
|
static |
Definition at line 20662 of file X86ISelLowering.cpp.
References llvm::X86ISD::BSR, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::ISD::CTLZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::isVector(), LowerVectorCTLZ(), llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 22624 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::MVT::isVector(), and LowerVectorCTPOP().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 20704 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::X86ISD::BSF, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::ISD::CTLZ, llvm::ISD::CTPOP, llvm::ISD::CTTZ, llvm::ISD::CTTZ_ZERO_UNDEF, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::isVector(), and llvm::ISD::SUB.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 17474 of file X86ISelLowering.cpp.
References assert(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::tgtok::In, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::X86ISD::UNPCKL, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v64i8, llvm::X86ISD::VSEXT, llvm::X86ISD::VSRAI, llvm::X86ISD::VZEXT, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 17663 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::LoadSDNode::getExtensionType(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::X86Subtarget::hasVLX(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::EVT::isVector(), llvm::MipsISD::Lo, llvm::SPII::Load, llvm::MVT::Other, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SIGN_EXTEND, llvm::ISD::TokenFactor, llvm::MVT::v16i1, llvm::MVT::v16i8, llvm::MVT::v32i8, llvm::MVT::v8i1, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
Referenced by LowerExtendedLoad().
|
static |
Definition at line 17762 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::ISD::EXTLOAD, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getExtLoad(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SelectionDAG::getSignExtendVectorInReg(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), i, llvm::MVT::i1, llvm::MVT::i8, llvm::ISD::INSERT_VECTOR_ELT, llvm::MVT::integer_valuetypes(), llvm::MVT::isInteger(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::MVT::isVector(), llvm::SPII::Load, LowerExtended1BitVectorLoad(), llvm::MVT::Other, Ptr, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::TokenFactor, and llvm::X86ISD::VSEXT.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 13936 of file X86ISelLowering.cpp.
References assert(), extract128BitVector(), extract256BitVector(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasAVX(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), and llvm_unreachable.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 13513 of file X86ISelLowering.cpp.
References Assert, llvm::ISD::AssertZext, llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f32, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::is128BitVector(), llvm::isNullConstant(), llvm::X86ISD::PEXTRB, llvm::ISD::STORE, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), and llvm::MVT::v4i32.
|
static |
The only differences between FABS and FNEG are the mask and the logic op.
FNEG also has a folding opportunity for FNEG(FABS(x)).
Definition at line 15660 of file X86ISelLowering.cpp.
References llvm::lltok::APFloat, assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f128, llvm::MVT::f64, llvm::ISD::FABS, llvm::X86ISD::FAND, llvm::ISD::FNEG, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::APInt::getSignBit(), llvm::APInt::getSignedMaxValue(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEquad(), llvm::APFloatBase::IEEEsingle(), llvm::MVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SDNode::uses(), llvm::MVT::v2f64, and llvm::MVT::v4f32.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 15727 of file X86ISelLowering.cpp.
References llvm::lltok::APFloat, assert(), llvm::MVT::bitsGT(), llvm::MVT::bitsLT(), llvm::APFloat::clearSign(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f128, llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::APInt::getSignBit(), llvm::SDValue::getSimpleValueType(), llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEquad(), llvm::APFloatBase::IEEEsingle(), llvm::MVT::isVector(), llvm::APIntOps::Or(), llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v16f32, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v8f32, and llvm::MVT::v8f64.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 15797 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::MVT::f32, llvm::MVT::f64, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::i32, llvm::X86ISD::MOVMSK, llvm::ISD::SCALAR_TO_VECTOR, llvm::MVT::v2f64, and llvm::MVT::v4f32.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 15645 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::v2f32, llvm::MVT::v4f32, and llvm::X86ISD::VFPEXT.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 22853 of file X86ISelLowering.cpp.
References AMDGPU::RuntimeMD::KeyName::Args, assert(), llvm::CallingConv::C, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f64, llvm::StructType::get(), llvm::VectorType::get(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getExternalSymbol(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getTypeForEVT(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::is64Bit(), llvm::TargetLowering::ArgListEntry::isSExt, llvm::X86Subtarget::isTargetDarwin(), llvm::TargetLowering::ArgListEntry::isZExt, llvm::TargetLowering::LowerCallTo(), llvm::ISD::MERGE_VALUES, llvm::TargetLowering::ArgListEntry::Node, llvm::TargetLowering::CallLoweringInfo::setChain(), llvm::TargetLowering::CallLoweringInfo::setDebugLoc(), and llvm::TargetLowering::ArgListEntry::Ty.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Compute the horizontal sum of bytes in V for the elements of VT.
Requires V to be a byte vector and VT to be an integer vector type with wider elements than V's type. The width of the elements of VT determines how many bytes of V are summed horizontally to produce each element of the result.
Definition at line 22395 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getZeroVector(), High, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::X86ISD::PACKUS, llvm::X86ISD::PSADBW, llvm::ISD::SHL, llvm::ISD::SRL, llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by LowerVectorCTPOPBitmath(), and LowerVectorCTPOPInRegLUT().
|
static |
Definition at line 13969 of file X86ISelLowering.cpp.
References llvm::TargetLoweringBase::allowsMemoryAccess(), areOnlyUsersOf(), assert(), llvm::dyn_cast(), EltsFromConsecutiveLoads(), llvm::CallingConv::Fast, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::X86Subtarget::getTargetLowering(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX(), llvm::MVT::i1, insert128BitVector(), insert1BitVector(), insert256BitVector(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isNormalLoad(), llvm_unreachable, peekThroughBitcasts(), peekThroughOneUseBitcasts(), and llvm::X86ISD::SUBV_BROADCAST.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 19883 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::ADX, assert(), llvm::X86ISD::CMOV, llvm::COMPRESS_TO_MEM, llvm::X86::COND_B, llvm::X86::COND_NE, llvm::MachineFrameInfo::CreateFixedObject(), llvm::Depth, llvm::dyn_cast(), EmitMaskedTruncSStore(), EmitTruncSStore(), llvm::EXPAND_FROM_MEM, llvm::GATHER, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), getExtendedControlRegister(), llvm::X86MachineFunctionInfo::getFAIndex(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), getGatherNode(), llvm::MachineFunction::getInfo(), llvm::getIntrinsicWithChain(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMaskedStore(), getMaskNode(), llvm::TargetMachine::getMCAsmInfo(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), getPrefetchNode(), getReadPerformanceCounter(), getReadTimeStampCounter(), llvm::X86Subtarget::getRegisterInfo(), llvm::X86TargetLowering::getReturnAddressFrameIndex(), getScatterNode(), getSETCC(), llvm::SDValue::getSimpleValueType(), llvm::X86RegisterInfo::getSlotSize(), llvm::SelectionDAG::getStore(), llvm::MachineFunction::getTarget(), llvm::SelectionDAG::getTruncStore(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::Glue, llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::isAllOnesConstant(), isValid(), llvm::X86::isZeroNode(), llvm_unreachable, MarkEHGuard(), MarkEHRegistrationNode(), llvm::ISD::MERGE_VALUES, llvm::ISD::NON_EXTLOAD, llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, llvm::MVT::Other, llvm::PREFETCH, llvm::RDPMC, llvm::RDRAND, llvm::RDSEED, llvm::RDTSC, Results, llvm::MipsISD::Ret, llvm::SCATTER, llvm::X86MachineFunctionInfo::setFAIndex(), llvm::MachineFrameInfo::setFrameAddressIsTaken(), llvm::MachineFrameInfo::setHasCopyImplyingStackAdjustment(), llvm::MachineFrameInfo::setReturnAddressIsTaken(), llvm::TRUNCATE_TO_MEM_VI16, llvm::TRUNCATE_TO_MEM_VI32, llvm::TRUNCATE_TO_MEM_VI8, llvm::IntrinsicData::Type, llvm::MCAsmInfo::usesWindowsCFI(), llvm::TargetLowering::verifyReturnAddressArgumentIsConstant(), llvm::X86ISD::VTRUNC, llvm::X86ISD::VTRUNCS, llvm::X86ISD::VTRUNCUS, llvm::XGETBV, llvm::XTEST, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 18785 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::BRCST32x2_TO_VEC, llvm::BRCST_SUBVEC_TO_VEC, llvm::BROADCASTM, llvm::CMP_MASK, llvm::CMP_MASK_CC, llvm::CMP_MASK_SCALAR_CC, llvm::COMI, llvm::COMI_RM, llvm::COMPRESS_EXPAND_IN_REG, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_O, llvm::X86::COND_P, llvm::X86::COND_S, llvm::CONVERT_MASK_TO_VEC, llvm::CONVERT_TO_MASK, llvm::X86::CUR_DIRECTION, llvm::CVTPD2PS, llvm::CVTPD2PS_MASK, llvm::dyn_cast(), llvm::MVT::f32, llvm::MVT::f64, llvm::FIXUPIMM, llvm::FIXUPIMM_MASKZ, llvm::FIXUPIMMS, llvm::FIXUPIMMS_MASKZ, llvm::FMA_OP_MASK, llvm::FMA_OP_MASK3, llvm::FMA_OP_MASKZ, llvm::FMA_OP_SCALAR_MASK, llvm::FMA_OP_SCALAR_MASK3, llvm::FMA_OP_SCALAR_MASKZ, llvm::ISD::FP_ROUND, llvm::FPCLASS, llvm::FPCLASSS, llvm::X86ISD::FSETCCM, llvm::X86ISD::FSETCCM_RND, llvm::X86RegisterInfo::getBaseRegister(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MachineModuleInfo::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MachineFunction::getFunctionNumber(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::getIntrinsicWithoutChain(), llvm::SelectionDAG::getMachineFunction(), getMaskNode(), llvm::SelectionDAG::getMCSymbol(), llvm::MachineFunction::getMMI(), llvm::Value::getName(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOperand(), llvm::MCContext::getOrCreateLSDASymbol(), llvm::TargetLoweringBase::getPointerTy(), llvm::X86RegisterInfo::getPtrSizedFrameRegister(), llvm::GlobalValue::getRealLinkageName(), llvm::X86Subtarget::getRegisterInfo(), getScalarMaskingNode(), llvm::MVT::getScalarType(), getSETCC(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), getVectorMaskingNode(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::X86RegisterInfo::hasBasePointer(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::INTR_TYPE_1OP, llvm::INTR_TYPE_1OP_MASK, llvm::INTR_TYPE_1OP_MASK_RM, llvm::INTR_TYPE_2OP, llvm::INTR_TYPE_2OP_IMM8_MASK, llvm::INTR_TYPE_2OP_MASK, llvm::INTR_TYPE_2OP_MASK_RM, llvm::INTR_TYPE_3OP, llvm::INTR_TYPE_3OP_IMM8_MASK, llvm::INTR_TYPE_3OP_MASK, llvm::INTR_TYPE_3OP_MASK_RM, llvm::INTR_TYPE_3OP_SCALAR_MASK_RM, llvm::INTR_TYPE_4OP, llvm::INTR_TYPE_SCALAR_MASK, llvm::INTR_TYPE_SCALAR_MASK_RM, llvm::MVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::isAllOnesConstant(), llvm::X86ISD::KORTEST, llvm::KUNPCK, LLVM_FALLTHROUGH, llvm_unreachable, llvm::BitmaskEnumDetail::Mask(), llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, llvm::ISD::OR, llvm::X86ISD::PCMPESTRI, llvm::X86ISD::PCMPISTRI, llvm::X86ISD::PTEST, recoverFramePointer(), llvm::report_fatal_error(), llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::TERLOG_OP_MASK, llvm::TERLOG_OP_MASKZ, llvm::X86ISD::TESTP, llvm::ISD::TRUNCATE, llvm::IntrinsicData::Type, llvm::MVT::v16i1, llvm::VPERM_2OP_MASK, llvm::VPERM_3OP_MASK, llvm::VPERM_3OP_MASKZ, llvm::X86ISD::VPERMV, llvm::VSHIFT, llvm::X86ISD::Wrapper, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Definition at line 16610 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::CMPM, llvm::X86ISD::CMPMU, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::i1, llvm::MVT::i8, LLVM_FALLTHROUGH, llvm_unreachable, llvm::X86ISD::PCMPEQM, llvm::X86ISD::PCMPGTM, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, std::swap(), and Unsigned.
Referenced by LowerVSETCC().
|
static |
Lowers masks values (v*i1) to the local register values.
Definition at line 2104 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::ISD::SIGN_EXTEND, llvm::MVT::v16i1, llvm::MVT::v32i1, llvm::MVT::v64i1, and llvm::MVT::v8i1.
|
static |
Definition at line 23159 of file X86ISelLowering.cpp.
References assert(), ExtendToType(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMaskedGather(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValue(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::is512BitVector(), llvm::BitmaskEnumDetail::Mask(), llvm::MVT::Other, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::SelectionDAG::UpdateNodeOperands(), and llvm::MVT::v8i64.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 20825 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::MVT::is256BitVector(), llvm::MVT::isInteger(), and Lower256IntArith().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 23051 of file X86ISelLowering.cpp.
References assert(), ExtendToType(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::MaskedLoadStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MaskedLoadSDNode::getSrc0(), llvm::SDValue::getValue(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i8, llvm::MaskedLoadSDNode::isExpandingLoad(), llvm::BitmaskEnumDetail::Mask(), and llvm::ISD::TRUNCATE.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 22957 of file X86ISelLowering.cpp.
References assert(), ExtendToType(), llvm::MaskedGatherScatterSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::MaskedGatherScatterSDNode::getIndex(), llvm::MaskedGatherScatterSDNode::getMask(), llvm::SelectionDAG::getMaskedScatter(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::MaskedGatherScatterSDNode::getValue(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::is512BitVector(), llvm::BitmaskEnumDetail::Mask(), llvm::MVT::Other, llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, llvm::MVT::v2i1, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i32, and llvm::MVT::v8i64.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 23110 of file X86ISelLowering.cpp.
References assert(), ExtendToType(), llvm::MaskedLoadStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MaskedStoreSDNode::getValue(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i8, llvm::MVT::is512BitVector(), llvm::MaskedStoreSDNode::isCompressingStore(), llvm::MaskedStoreSDNode::isTruncatingStore(), llvm::BitmaskEnumDetail::Mask(), and llvm::ISD::TRUNCATE.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 20832 of file X86ISelLowering.cpp.
References A, llvm::ISD::ADD, llvm::ISD::AND, assert(), B, llvm::SelectionDAG::ComputeNumSignBits(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetVShiftByConstNode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::is256BitVector(), Lower256IntArith(), Lower512IntArith(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::ISD::MUL, llvm::X86ISD::PACKUS, llvm::X86ISD::PMULDQ, llvm::X86ISD::PMULUDQ, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::TRUNCATE, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v64i8, llvm::MVT::v8i16, llvm::MVT::v8i64, llvm::X86ISD::VSEXT, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLI, and Zero.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 21170 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::ISD::CONCAT_VECTORS, extract128BitVector(), Fixup, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::TargetLoweringBase::getShiftAmountTy(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::is256BitVector(), llvm::MipsISD::Lo, llvm::makeArrayRef(), llvm::X86ISD::PMULDQ, llvm::X86ISD::PMULUDQ, llvm::ISD::SMUL_LOHI, llvm::ISD::SRA, llvm::ISD::SUB, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v4i64, and llvm::MVT::v8i32.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 21004 of file X86ISelLowering.cpp.
References A, assert(), B, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::is256BitVector(), llvm::MipsISD::Lo, Lower256IntArith(), llvm::ISD::MUL, llvm::ISD::MULHU, llvm::X86ISD::PACKUS, llvm::ISD::SRA, llvm::ISD::SRL, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v32i8, llvm::MVT::v8i16, llvm::X86ISD::VSEXT, and llvm::X86ISD::VZEXT.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 19811 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), getReadTimeStampCounter(), llvm::X86ISD::RDTSC_DAG, and Results.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
The function will lower a register of various sizes (8/16/32/64) to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1)
Definition at line 2454 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::EVT::getSimpleVT(), llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm_unreachable, llvm::MVT::SimpleTy, llvm::ISD::TRUNCATE, llvm::MVT::v16i1, llvm::MVT::v32i1, llvm::MVT::v64i1, and llvm::MVT::v8i1.
|
static |
Definition at line 22014 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasXOP(), llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::isVector(), Lower256IntArith(), llvm::ISD::ROTL, llvm::X86ISD::VPROT, and llvm::X86ISD::VPROTI.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 13909 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::i32, insert128BitVector(), llvm::MVT::is128BitVector(), llvm::ISD::SCALAR_TO_VECTOR, and llvm::MVT::v4i32.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 21312 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::ARMISD::CMP, llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), getTargetVShiftByConstNode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::ConstantSDNode::getZExtValue(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasXOP(), i, llvm::MVT::i32, llvm::MVT::is512BitVector(), llvm::X86Subtarget::is64Bit(), llvm_unreachable, llvm::Log2_32_Ceil(), llvm::X86ISD::PCMPGT, llvm::X86ISD::PCMPGTM, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, SupportedVectorShiftWithImm(), llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4i64, llvm::MVT::v64i1, llvm::MVT::v64i8, llvm::MVT::v8i64, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and llvm::ISD::XOR.
Referenced by LowerShift().
|
static |
Definition at line 21483 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::MVT::bitsGT(), llvm::MVT::bitsLE(), llvm::MVT::bitsLT(), llvm::ISD::BUILD_VECTOR, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::ShuffleVectorSDNode::getSplatIndex(), getTargetVShiftNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), i, llvm::MVT::i32, llvm::MVT::i64, llvm::ISD::INSERT_VECTOR_ELT, llvm::X86Subtarget::is64Bit(), llvm::ShuffleVectorSDNode::isSplat(), llvm::SDValue::isUndef(), llvm::ISD::SHL, llvm::ISD::SRL, SupportedVectorShiftWithBaseAmnt(), llvm::MVT::v2i64, Vals, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerShift().
|
static |
Definition at line 21566 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), C, llvm::ISD::FP_TO_SINT, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::APInt::getSignBit(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasXOP(), i, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDNode::isUndef(), llvm::MVT::isVector(), LowerScalarImmediateShift(), LowerScalarVariableShift(), llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, llvm::ISD::MUL, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ISD::SHL, llvm::APInt::shl(), llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, SupportedVectorVarShift(), llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8i16, llvm::X86ISD::VPSHA, llvm::X86ISD::VPSHL, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount.
Definition at line 14510 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::X86ISD::CMOV, llvm::X86ISD::CMP, llvm::X86::COND_NE, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::MVT::i8, llvm::MipsISD::Lo, llvm::ISD::SHL, llvm::ISD::SHL_PARTS, llvm::X86ISD::SHLD, llvm::X86ISD::SHRD, llvm::ISD::SRA, llvm::ISD::SRA_PARTS, and llvm::ISD::SRL.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes.
Definition at line 11924 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), i, is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), and llvm::SM_SentinelUndef.
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Definition at line 17553 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), i, llvm::MVT::i1, llvm::MVT::is512BitVector(), LowerSIGN_EXTEND_AVX512(), llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8i16, llvm::MVT::v8i32, and llvm::X86ISD::VSEXT.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 17414 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::is512BitVector(), llvm::ISD::VSELECT, llvm::X86ISD::VSEXT, llvm::X86ISD::VTRUNC, and llvm::X86ISD::VZEXT.
Referenced by LowerSIGN_EXTEND().
|
static |
Definition at line 20815 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), llvm::MVT::i1, llvm::MVT::is256BitVector(), llvm::MVT::isInteger(), Lower256IntArith(), and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB node.
Definition at line 7120 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADDSUB, llvm::X86ISD::FMADDSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getSimpleValueType(), llvm::MVT::is512BitVector(), isAddSub(), and isFMAddSub().
|
static |
Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
Definition at line 7147 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, ExpandHorizontalBinOp(), llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, i, isHorizontalBinOp(), llvm::SDValue::isUndef(), llvm::SDNode::isUndef(), llvm::ISD::SUB, llvm::MVT::v16i16, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::MVT::v8f32, llvm::MVT::v8i16, and llvm::MVT::v8i32.
|
static |
Definition at line 14307 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getEntryNode(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachinePointerInfo::getGOT(), llvm::Type::getInt8PtrTy(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SDNode::getValueType(), llvm::X86ISD::GlobalBaseReg, llvm::TLSModel::InitialExec, llvm_unreachable, llvm::TLSModel::LocalExec, llvm::X86II::MO_GOTNTPOFF, llvm::X86II::MO_GOTTPOFF, llvm::X86II::MO_INDNTPOFF, llvm::X86II::MO_NTPOFF, llvm::X86II::MO_TPOFF, Offset, Ptr, llvm::MipsISD::ThreadPointer, llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.
|
static |
Definition at line 14247 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), GetTLSADDR(), llvm::SDValue::getValue(), llvm::X86ISD::GlobalBaseReg, and llvm::X86II::MO_TLSGD.
|
static |
Definition at line 14261 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getEntryNode(), GetTLSADDR(), and llvm::X86II::MO_TLSGD.
|
static |
Definition at line 14267 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getEntryNode(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachineFunction::getInfo(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), GetTLSADDR(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::X86ISD::GlobalBaseReg, llvm::X86MachineFunctionInfo::incNumLocalDynamicTLSAccesses(), llvm::X86II::MO_DTPOFF, llvm::X86II::MO_TLSLD, llvm::X86II::MO_TLSLDM, and llvm::X86ISD::Wrapper.
|
static |
Definition at line 16467 of file X86ISelLowering.cpp.
References assert(), getBitTestCondition(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::MVT::i1, llvm::ISD::SRL, and llvm::ISD::TRUNCATE.
|
static |
Definition at line 15429 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::CVT2MASK, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, and llvm::X86ISD::TESTM.
|
static |
Definition at line 17605 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getUNDEF(), llvm::StoreSDNode::getValue(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::X86Subtarget::hasVLX(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::ISD::INSERT_SUBVECTOR, llvm::StoreSDNode::isTruncatingStore(), llvm::EVT::isVector(), llvm::MipsISD::Lo, llvm::MVT::Other, llvm::ISD::TokenFactor, llvm::ISD::TRUNCATE, llvm::MVT::v16i1, llvm::MVT::v16i8, llvm::MVT::v32i8, and llvm::MVT::v8i1.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 14793 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTUI2P, llvm::ISD::FADD, llvm::ISD::FMUL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasAVX512(), llvm::AArch64CC::HI, llvm::AArch64CC::LO, llvm::ISD::SRL, llvm::MVT::v2f64, llvm::MVT::v2i32, and llvm::MVT::v4i32.
|
static |
Definition at line 14828 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::lltok::APFloat, assert(), llvm::X86ISD::BLENDI, llvm::ISD::FADD, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getTarget(), llvm::X86Subtarget::hasSSE41(), High, llvm::MVT::i32, llvm::APFloatBase::IEEEsingle(), llvm::TargetMachine::Options, llvm::ISD::OR, llvm::ISD::SRL, llvm::TargetOptions::UnsafeFPMath, llvm::MVT::v16i16, llvm::MVT::v4f32, llvm::MVT::v4i32, llvm::MVT::v8f32, llvm::MVT::v8i16, and llvm::MVT::v8i32.
|
static |
Handle lowering of 16-lane 32-bit floating point shuffles.
Definition at line 12898 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), llvm::SDValue::isUndef(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithSHUFPS(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::ArrayRef< T >::size(), llvm::MVT::v16f32, and llvm::X86ISD::VPERMILPI.
Referenced by lower512BitVectorShuffle().
|
static |
Handle lowering of 16-lane 16-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v16i16 shuffling..
Definition at line 12535 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVLX(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::SDValue::isUndef(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithPSHUFB(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFB, llvm::ArrayRef< T >::size(), and llvm::MVT::v16i16.
Referenced by lower256BitVectorShuffle().
|
static |
Handle lowering of 16-lane 32-bit integer shuffles.
Definition at line 13001 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasBWI(), is128BitLaneRepeatedShuffleMask(), isSingleSHUFPSMask(), llvm::SDValue::isUndef(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithSHUFPS(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::MVT::v16f32, and llvm::MVT::v16i32.
Referenced by lower512BitVectorShuffle().
|
static |
Generic lowering of v16i8 shuffles.
This is a hybrid strategy to lower v16i8 vectors. It first attempts to detect any complexity reducing interleaving. If that doesn't help, it uses UNPCK to spread the i8 elements across two i16-element vectors, and uses the existing lowering for v8i16 blends on each half, finally PACK-ing them back together.
Definition at line 11077 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), canLowerByDroppingEvenElements(), llvm::count_if(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), I, i, llvm::SDValue::isUndef(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBlendOfPSHUFBs(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPermuteAndUnpack(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), lowerVectorShuffleWithUNPCK(), llvm::BitmaskEnumDetail::Mask(), llvm::none_of(), llvm::X86ISD::PACKUS, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, and Zero.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering of 2-lane 64-bit floating point shuffles.
This is the basis function for the 2-lane 64-bit shuffles as we have full support for floating point shuffles but not integer shuffles. These instructions will incur a domain crossing penalty on some chips though so it is better to avoid lowering through this for integer vectors where possible.
Definition at line 9879 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getScalarValueForVectorElement(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::i8, isShuffleEquivalent(), isShuffleFoldableLoad(), llvm::SDValue::isUndef(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVLPD, llvm::X86ISD::MOVSD, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::SHUFP, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::MVT::v2f64, and llvm::X86ISD::VPERMILPI.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering of 2-lane 64-bit integer shuffles.
Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by the integer unit to minimize domain crossing penalties. However, for blends it falls back to the floating point shuffle operation with appropriate bit casting.
Definition at line 9961 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::SDValue::isUndef(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsShift(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PACKUS, peekThroughBitcasts(), llvm::X86ISD::PSHUFD, llvm::ArrayRef< T >::size(), llvm::MVT::v2f64, llvm::MVT::v2i64, and llvm::MVT::v4i32.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering 2-lane 128-bit shuffles.
Definition at line 11622 of file X86ISelLowering.cpp.
References canWidenShuffleElements(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::i8, llvm::ISD::isBuildVectorAllZeros(), isShuffleEquivalent(), llvm::SDValue::isUndef(), lowerVectorShuffleAsBlend(), and llvm::X86ISD::VPERM2X128.
Referenced by lowerV4F64VectorShuffle(), and lowerV4I64VectorShuffle().
|
static |
Handle lowering of 32-lane 16-bit integer shuffles.
Definition at line 13069 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasBWI(), is128BitLaneRepeatedShuffleMask(), llvm::SDValue::isUndef(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithUNPCK(), llvm::ArrayRef< T >::size(), and llvm::MVT::v32i16.
Referenced by lower512BitVectorShuffle().
|
static |
Handle lowering of 32-lane 8-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v32i8 shuffling..
Definition at line 12621 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasAVX2(), is128BitLaneCrossingShuffleMask(), llvm::SDValue::isUndef(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleWithPSHUFB(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFB, llvm::ArrayRef< T >::size(), and llvm::MVT::v32i8.
Referenced by lower256BitVectorShuffle().
|
static |
Lower 4-lane 32-bit floating point shuffles.
Uses instructions exclusively from the floating point unit to minimize domain crossing penalties, as these are sufficient to implement all v4f32 shuffles.
Definition at line 10180 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), isShuffleEquivalent(), isSingleSHUFPSMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBlendAndPermute(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsInsertPS(), lowerVectorShuffleWithSHUFPS(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::SHUFP, llvm::ArrayRef< T >::size(), llvm::MVT::v4f32, and llvm::X86ISD::VPERMILPI.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering of 4-lane 64-bit floating point shuffles.
Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2 isn't available.
Definition at line 12155 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i8, is128BitLaneCrossingShuffleMask(), isShuffleEquivalent(), isShuffleMaskInputInPlace(), llvm::SDValue::isUndef(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV2X128VectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithSHUFPD(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVDDUP, llvm::ArrayRef< T >::size(), llvm::MVT::v4f64, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by lower256BitVectorShuffle().
|
static |
Lower 4-lane i32 vector shuffles.
We try to handle these with integer-domain shuffles where we can, but for blends we use the floating point domain blend instructions.
Definition at line 10263 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), isShuffleEquivalent(), isSingleSHUFPSMask(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPermuteAndUnpack(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, llvm::ArrayRef< T >::size(), llvm::MVT::v4f32, and llvm::MVT::v4i32.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering of 4-lane 64-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v4i64 shuffling..
Definition at line 12252 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasVLX(), is128BitLaneRepeatedShuffleMask(), isShuffleMaskInputInPlace(), llvm::SDValue::isUndef(), lowerV2X128VectorShuffle(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, scaleShuffleMask(), llvm::ArrayRef< T >::size(), llvm::MVT::v4i64, llvm::MVT::v8i32, and llvm::X86ISD::VPERMI.
Referenced by lower256BitVectorShuffle().
|
static |
Try to lower a vector shuffle as a 128-bit shuffles.
Definition at line 12759 of file X86ISelLowering.cpp.
References assert(), canWidenShuffleElements(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), i, llvm::MVT::i8, insert128BitVector(), llvm::MVT::is512BitVector(), isShuffleEquivalent(), and llvm::X86ISD::SHUF128.
Referenced by lowerV8F64VectorShuffle(), and lowerV8I64VectorShuffle().
|
static |
Handle lowering of 64-lane 8-bit integer shuffles.
Definition at line 13116 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVBMI(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithPSHUFB(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFB, llvm::ArrayRef< T >::size(), splitAndLowerVectorShuffle(), and llvm::MVT::v64i8.
Referenced by lower512BitVectorShuffle().
|
static |
Handle lowering of 8-lane 32-bit floating point shuffles.
Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2 isn't available.
Definition at line 12340 of file X86ISelLowering.cpp.
References assert(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasVLX(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), llvm::SDValue::isUndef(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsLanePermuteAndBlend(), lowerVectorShuffleAsSplitOrBlend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithSHUFPS(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::MVT::v8f32, llvm::MVT::v8i32, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, and llvm::X86ISD::VPERMV.
Referenced by lower256BitVectorShuffle().
|
static |
Handle lowering of 8-lane 64-bit floating point shuffles.
Definition at line 12846 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::MVT::i8, is128BitLaneCrossingShuffleMask(), is256BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), llvm::SDValue::isUndef(), lowerV4X128VectorShuffle(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithSHUFPD(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::MOVDDUP, llvm::ArrayRef< T >::size(), llvm::MVT::v8f64, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by lower512BitVectorShuffle().
|
static |
Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.
The lowering strategy is to try to form pairs of input lanes which are targeted at the same half of the final vector, and then use a dword shuffle to place them onto the right half, and finally unpack the paired lanes into their final position.
The exact breakdown of how to form these dword pairs and align them on the correct sides is really tricky. See the comments within the function for more of the details.
This code also handles repeated 128-bit lanes of v8i16 shuffles, but each lane must shuffle the exact same way. In fact, you must pass a v8 Mask to this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16 vector, form the analogous 128-bit 8-element Mask.
Definition at line 10381 of file X86ISelLowering.cpp.
References assert(), llvm::sys::path::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), llvm::MutableArrayRef< T >::begin(), llvm::count(), llvm::count_if(), llvm::SmallVectorTemplateCommon< T, typename >::data(), llvm::ArrayRef< T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::MutableArrayRef< T >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::find(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), i, llvm::MVT::i16, llvm::MVT::i32, llvm::is_contained(), isNoopShuffleMask(), llvm_unreachable, llvm::BitmaskEnumDetail::Mask(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::MutableArrayRef< T >::slice(), std::swap(), and llvm::MVT::v8i16.
Referenced by lowerV16I16VectorShuffle(), lowerV32I16VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Generic lowering of 8-lane i16 shuffles.
This handles both single-input shuffles and combined shuffle/blends with two inputs. The single input shuffles are immediately delegated to a dedicated lowering routine.
The blends are lowered in one of three fundamental ways. If there are few enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle of the input is significantly cheaper when lowered as an interleaving of the two inputs, try to interleave them. Otherwise, blend the low and high halves of the inputs separately (making them have relatively few inputs) and then concatenate them.
Definition at line 10893 of file X86ISelLowering.cpp.
References llvm::any_of(), assert(), llvm::ArrayRef< T >::begin(), llvm::count_if(), llvm::ArrayRef< T >::end(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), lowerV8I16GeneralSingleInputVectorShuffle(), lowerVectorShuffleAsBitBlend(), lowerVectorShuffleAsBitMask(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBlendOfPSHUFBs(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsElementInsertion(), lowerVectorShuffleAsPermuteAndUnpack(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleWithSSE4A(), lowerVectorShuffleWithUNPCK(), llvm::ArrayRef< T >::size(), and llvm::MVT::v8i16.
Referenced by lower128BitVectorShuffle().
|
static |
Handle lowering of 8-lane 32-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v8i32 shuffling..
Definition at line 12431 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasVLX(), is128BitLaneRepeatedShuffleMask(), isSingleSHUFPSMask(), llvm::SDValue::isUndef(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerVectorShuffleAsBlend(), lowerVectorShuffleAsBroadcast(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsDecomposedShuffleBlend(), lowerVectorShuffleAsRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleAsZeroOrAnyExtend(), lowerVectorShuffleByMerging128BitLanes(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithSHUFPS(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ArrayRef< T >::size(), llvm::MVT::v8f32, llvm::MVT::v8i32, and llvm::X86ISD::VPERMV.
Referenced by lower256BitVectorShuffle().
|
static |
Handle lowering of 8-lane 64-bit integer shuffles.
Definition at line 12940 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), is128BitLaneRepeatedShuffleMask(), is256BitLaneRepeatedShuffleMask(), llvm::SDValue::isUndef(), lowerV4X128VectorShuffle(), lowerVectorShuffleAsByteRotate(), lowerVectorShuffleAsRotate(), lowerVectorShuffleAsShift(), lowerVectorShuffleToEXPAND(), lowerVectorShuffleWithPERMV(), lowerVectorShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, scaleShuffleMask(), llvm::ArrayRef< T >::size(), llvm::MVT::v16i32, llvm::MVT::v8i64, and llvm::X86ISD::VPERMI.
Referenced by lower512BitVectorShuffle().
|
static |
Definition at line 18442 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::expandVACopy(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemcpy(), llvm::SDValue::getNode(), llvm::SDValue::getOperand(), llvm::X86Subtarget::is64Bit(), and llvm::X86Subtarget::isCallingConvWin64().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 15816 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorTemplateCommon< T >::back(), llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::begin(), llvm::SmallVectorTemplateCommon< T >::begin(), E, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE41(), I, i, llvm::MVT::i32, llvm::DenseMapBase< DenseMap< KeyT, ValueT, KeyInfoT, BucketT >, KeyT, ValueT, KeyInfoT, BucketT >::insert(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::ISD::OR, llvm::MVT::Other, llvm::X86ISD::PTEST, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::SmallVectorTemplateCommon< T >::size(), llvm::MVT::v2i64, and llvm::MVT::v4i64.
|
static |
Attempt to use the vbroadcast instruction to generate a splat value for the following cases:
The VBROADCAST node is returned when a pattern is found, or SDValue() otherwise.
Definition at line 6422 of file X86ISelLowering.cpp.
References assert(), llvm::APInt::bitsToDouble(), llvm::APInt::bitsToFloat(), llvm::ISD::Constant, llvm::ISD::ConstantFP, llvm::BitVector::count(), llvm::HexagonISD::CP, llvm::ConstantFP::get(), getAlignment(), llvm::SelectionDAG::getBitcast(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getConstantPool(), getConstantVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::Type::getDoubleTy(), llvm::SelectionDAG::getEntryNode(), llvm::MVT::getFloatingPointVT(), llvm::Type::getFloatTy(), llvm::MachineFunction::getFunction(), llvm::Constant::getIntegerValue(), llvm::MVT::getIntegerVT(), llvm::Type::getIntNTy(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::TargetLoweringBase::getPointerTy(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::BuildVectorSDNode::getSplatValue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasVLX(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::X86Subtarget::is32Bit(), llvm::MVT::is512BitVector(), llvm::BuildVectorSDNode::isConstantSplat(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::SDNode::isOnlyUserOf(), isUseOfShuffle(), llvm::EVT::isVector(), llvm::Function::optForSize(), llvm::X86ISD::SUBV_BROADCAST, llvm::RegState::Undef, and llvm::X86ISD::VBROADCAST.
|
static |
Definition at line 20636 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::CTLZ, extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::is256BitVector(), LowerVectorCTLZ_AVX512(), and LowerVectorCTLZInRegLUT().
Referenced by LowerCTLZ().
|
static |
Lower a vector CTLZ using native supported vector CTLZ instruction.
Definition at line 20514 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::CTLZ, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MipsISD::Lo, llvm::SelectionDAG::SplitVector(), llvm::ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerVectorCTLZ().
|
static |
Definition at line 20567 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MipsISD::Hi, i, llvm::MVT::i8, llvm::MipsISD::Lo, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ISD::SETEQ, and llvm::ISD::SRL.
Referenced by LowerVectorCTLZ().
|
static |
Definition at line 22583 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extract128BitVector(), extract256BitVector(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerVectorCTPOPBitmath(), and LowerVectorCTPOPInRegLUT().
Referenced by LowerCTPOP().
|
static |
Definition at line 22516 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::APInt::getSplat(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::MVT::i8, llvm::MVT::is128BitVector(), LowerHorizontalByteSum(), llvm::BitmaskEnumDetail::Mask(), llvm::ISD::SRL, and llvm::ISD::SUB.
Referenced by LowerVectorCTPOP().
|
static |
Definition at line 22459 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), i, llvm::MVT::i8, LowerHorizontalByteSum(), llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), and llvm::ISD::SRL.
Referenced by LowerVectorCTPOP().
|
static |
Top-level lowering for x86 vector shuffles.
This handles decomposition, canonicalization, and lowering of all x86 vector shuffles. Most of the specific lowering strategies are encapsulated above in helper routines. The canonicalization attempts to widen shuffles to involve fewer lanes of wider elements, consolidate symmetric patterns s.t. only one of the two inputs needs to be tested, etc.
Definition at line 13348 of file X86ISelLowering.cpp.
References llvm::SmallBitVector::all(), llvm::all_of(), assert(), canonicalizeShuffleMaskWithCommute(), canWidenShuffleElements(), computeZeroableShuffleElements(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getCommutedVectorShuffle(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::ShuffleVectorSDNode::getMask(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::i1, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm_unreachable, lower128BitVectorShuffle(), lower1BitVectorShuffle(), lower256BitVectorShuffle(), lower512BitVectorShuffle(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Try to emit a blend instruction for a shuffle using bit math.
This is used as a fallback approach when first class blend instructions are unavailable. Currently it is only suitable for integer vectors, but could be generalized for floating point vectors if desirable.
Definition at line 8284 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), i, llvm::MVT::i64, llvm::MVT::isInteger(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ArrayRef< T >::size(), and Zero.
Referenced by lower256BitVectorShuffle(), lowerV16I8VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Try to emit a bitmask instruction for a shuffle.
This handles cases where we can model a blend exactly as a bitmask due to one of the inputs being zeroable.
Definition at line 8249 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), i, llvm::MVT::isFloatingPoint(), llvm::ArrayRef< T >::size(), llvm::NVPTX::PTXLdStInstCode::V2, and Zero.
Referenced by lower256BitVectorShuffle(), lowerV16I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerVectorShuffleAsBlend().
|
static |
Try to emit a blend instruction for a shuffle.
This doesn't do any checks for the availability of instructions for blending these values. It relies on the availability of the X86ISD::BLENDI pattern to be matched in the backend with the type given. What it does check for is that the shuffle mask is a blend, or convertible into a blend with zero.
Definition at line 8316 of file X86ISelLowering.cpp.
References assert(), llvm::ArrayRef< T >::begin(), llvm::X86ISD::BLENDI, llvm::ArrayRef< T >::end(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), i, llvm::MVT::i8, is128BitLaneRepeatedShuffleMask(), llvm::MVT::is128BitVector(), llvm::ISD::isBuildVectorAllZeros(), LLVM_FALLTHROUGH, llvm_unreachable, lowerVectorShuffleAsBitMask(), llvm::BitmaskEnumDetail::Mask(), llvm::SmallVectorTemplateBase< T, isPodLike >::push_back(), llvm::MVT::SimpleTy, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v32i8, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8f32, llvm::MVT::v8i16, llvm::MVT::v8i32, and llvm::ISD::VSELECT.
Referenced by lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV2X128VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Try to lower as a blend of elements from two inputs followed by a single-input permutation.
This matches the pattern where we can blend elements from two inputs and then reduce the shuffle to a single-input permutation.
Definition at line 8479 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), i, and llvm::ArrayRef< T >::size().
Referenced by lowerV4F32VectorShuffle(), and lowerVectorShuffleAsDecomposedShuffleBlend().
|
static |
Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used.
Definition at line 10831 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), i, llvm::MVT::i8, llvm::ISD::OR, llvm::X86ISD::PSHUFB, llvm::ArrayRef< T >::size(), llvm::MVT::v16i8, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16I8VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Try to lower broadcast of a single element.
For convenience, this code also bundles all of the subtarget feature set filtering. While a little annoying to re-dispatch on type here, there isn't a convenient way to factor it out. FIXME: This is very similar to LowerVectorBroadcast - can we merge them?
Definition at line 9473 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getStoreSize(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ConstantSDNode::getZExtValue(), llvm::SDNode::hasAnyUseOfValue(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE3(), i, llvm::MVT::i64, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), llvm::MVT::isInteger(), isShuffleEquivalent(), isShuffleFoldableLoad(), llvm::MVT::isVector(), isVolatile(), lowerVectorShuffleAsTruncBroadcast(), MayFoldLoad(), llvm::X86ISD::MOVDDUP, Offset, llvm::MVT::Other, peekThroughBitcasts(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ArrayRef< T >::size(), llvm::ISD::TokenFactor, llvm::SelectionDAG::UpdateNodeOperands(), llvm::MVT::v2f64, llvm::MVT::v4f64, llvm::MVT::v4i64, and llvm::X86ISD::VBROADCAST.
Referenced by lower512BitVectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Definition at line 8657 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSSE3(), llvm::MipsISD::Hi, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is512BitVector(), isNoopShuffleMask(), llvm::MipsISD::Lo, matchVectorShuffleAsByteRotate(), llvm::ISD::OR, llvm::X86ISD::PALIGNR, llvm::ArrayRef< T >::size(), llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.
Referenced by lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
|
static |
Generic routine to decompose a shuffle and blend into indepndent blends and permutes.
This matches the extremely common pattern for handling combined shuffle+blend operations on newer X86 ISAs where we have very fast blend operations. It will try to pick the best arrangement of shuffles and blends.
Definition at line 8513 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), i, isNoopShuffleMask(), lowerVectorShuffleAsBlendAndPermute(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerVectorShuffleAsSplitOrBlend().
|
static |
Try to lower insertion of a single element into a zero vector.
This is a common pattern that we have especially efficient patterns to lower across all subtarget feature sets.
Definition at line 9315 of file X86ISelLowering.cpp.
References assert(), llvm::ArrayRef< T >::begin(), llvm::ArrayRef< T >::end(), llvm::MVT::f32, llvm::MVT::f64, llvm::find_if(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getScalarShiftAmountTy(), getScalarValueForVectorElement(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), i, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, llvm::ISD::SCALAR_TO_VECTOR, llvm::ArrayRef< T >::size(), llvm::MVT::v16i8, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v4i32, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VZEXT_MOVL, and llvm::ISD::ZERO_EXTEND.
Referenced by lower256BitVectorShuffle(), lower512BitVectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4I32VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Definition at line 9743 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::i8, llvm::X86ISD::INSERTPS, matchVectorShuffleAsInsertPS(), and llvm::MVT::v4f32.
Referenced by lowerV4F32VectorShuffle().
|
static |
Lower a vector shuffle crossing multiple 128-bit lanes as a permutation and blend of those lanes.
This essentially blends the out-of-lane inputs to each lane into the lane from a permuted copy of the vector. This lowering strategy results in four instructions in the worst case for a single-input cross lane shuffle which is lower than any other fully general cross-lane shuffle strategy I'm aware of. Special cases for each particular shuffle pattern should be handled prior to trying this lowering.
Definition at line 11580 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), i, llvm::MVT::i8, llvm::MVT::is256BitVector(), llvm::SDValue::isUndef(), llvm::ArrayRef< T >::size(), splitAndLowerVectorShuffle(), and llvm::X86ISD::VPERM2X128.
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), and lowerV8F32VectorShuffle().
|
static |
Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.
This specifically targets cases where we end up with alternating between the two inputs, and so can permute them into something that feeds a single UNPCK instruction. Note that this routine only targets integer vectors because for floating point vectors we have a generalized SHUFPS lowering strategy that handles everything that doesn't exactly match an unpack, making this clever lowering unnecessary.
Definition at line 9769 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), i, llvm::MVT::is128BitVector(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), llvm::SDValue::isUndef(), llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by lowerV16I8VectorShuffle(), lowerV4I32VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Try to lower a vector shuffle as a dword/qword rotation.
AVX512 has a VALIGND/VALIGNQ instructions that will do an arbitrary rotation of the concatenation of two vectors; This routine will try to generically lower a vector shuffle through such an pattern.
Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.
Definition at line 8713 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::X86Subtarget::hasVLX(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MipsISD::Lo, matchVectorShuffleAsRotate(), llvm::NVPTX::PTXLdStInstCode::V2, and llvm::X86ISD::VALIGN.
Referenced by lowerV16I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
|
static |
Definition at line 8820 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::MVT::i8, llvm::TargetLoweringBase::isTypeLegal(), llvm::BitmaskEnumDetail::Mask(), matchVectorShuffleAsShift(), llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
|
static |
Lower a vector shuffle as a zero or any extension.
Given a specific number of elements, element bit width, and extension stride, produce either a zero or any extension based on the available features of the subtarget. The extended elements are consecutive and begin and can start from an offseted element index in the input; to avoid excess shuffling the offset must either being in the bottom lane or at the start of a higher lane. All extended elements must be from the same lane.
Definition at line 8994 of file X86ISelLowering.cpp.
References assert(), llvm::MipsISD::Ext, extractSubVector(), llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE4A(), llvm::X86Subtarget::hasSSSE3(), llvm::MipsISD::Hi, i, llvm::MVT::i8, llvm::MVT::is128BitVector(), isUndefInRange(), llvm::MipsISD::Lo, Offset, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::ArrayRef< T >::size(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, and llvm::X86ISD::VZEXT.
Referenced by lowerVectorShuffleAsZeroOrAnyExtend().
|
static |
Either split a vector in halves or decompose the shuffles and the blend.
This is provided as a good fallback for many lowerings of non-single-input shuffles with more than one 128-bit lane. In those cases, we want to select between splitting the shuffle into 128-bit components and stitching those back together vs. extracting the single-input shuffles and blending those results.
Definition at line 11521 of file X86ISelLowering.cpp.
References assert(), llvm::SmallBitVector::count(), llvm::count(), llvm::MVT::getSizeInBits(), i, llvm::SDValue::isUndef(), lowerVectorShuffleAsDecomposedShuffleBlend(), llvm::SmallBitVector::resize(), llvm::ArrayRef< T >::size(), and splitAndLowerVectorShuffle().
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), and lowerV8F32VectorShuffle().
|
static |
Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements.
This assumes we have AVX2.
Definition at line 9418 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::isInteger(), llvm::EVT::isVector(), llvm::NVPTX::PTXLdStInstCode::Scalar, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::X86ISD::VBROADCAST.
Referenced by lowerVectorShuffleAsBroadcast().
|
static |
Try to lower a vector shuffle as a zero extension on any microarch.
This routine will try to do everything in its power to cleverly lower a shuffle which happens to match the pattern of a zero extend. It doesn't check for the profitability of this lowering, it tries to aggressively match this pattern. It will use all of the micro-architectural details it can to emit an efficient lowering. It handles both blends with all-zero inputs to explicitly zero-extend and undef-lanes (sometimes undef due to masking out later).
The reason we have dedicated lowering for zext-style shuffles is that they are both incredibly common and often quite performance sensitive.
Definition at line 9159 of file X86ISelLowering.cpp.
References assert(), llvm::tgtok::Bits, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), i, isSequentialOrUndefInRange(), lowerVectorShuffleAsSpecificZeroOrAnyExtend(), llvm::ArrayRef< T >::size(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2i64, and llvm::X86ISD::VZEXT_MOVL.
Referenced by lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4I32VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8I16VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.
This will only succeed when the result of fixing the 128-bit lanes results in a single-input non-lane-crossing shuffle with a repeating shuffle mask in each 128-bit lanes. This handles many cases where we can quickly blend away the lane crosses early and then use simpler shuffles within each lane.
FIXME: It might be worthwhile at some point to support this without requiring the 128-bit lane-relative shuffles to be repeating, but currently in x86 only floating point has interesting non-repeating shuffles, and even those are still marginally more expensive.
Definition at line 11719 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::f64, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), i, llvm::MVT::i64, is128BitLaneCrossingShuffleMask(), llvm::MVT::isFloatingPoint(), llvm::SDValue::isUndef(), and llvm::ArrayRef< T >::size().
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Definition at line 8193 of file X86ISelLowering.cpp.
References assert(), convertBitVectorToUnsiged(), llvm::X86ISD::EXPAND, llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), getMaskNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::i1, isNonZeroElementsInOrder(), and llvm::ISD::VSELECT.
Referenced by lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I64VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
|
static |
Definition at line 12138 of file X86ISelLowering.cpp.
References getConstVector(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SDValue::isUndef(), llvm::X86ISD::VPERMV, and llvm::X86ISD::VPERMV3.
Referenced by lowerV16F32VectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV32I16VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F64VectorShuffle(), and lowerV8I64VectorShuffle().
|
static |
Try to lower a shuffle with a single PSHUFB of V1 or V2.
Definition at line 8124 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasSSSE3(), i, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::X86ISD::PSHUFB, and llvm::ArrayRef< T >::size().
Referenced by lowerV16I16VectorShuffle(), lowerV32I8VectorShuffle(), and lowerV64I8VectorShuffle().
|
static |
Definition at line 12127 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i8, matchVectorShuffleWithSHUFPD(), and llvm::X86ISD::SHUFP.
Referenced by lowerV4F64VectorShuffle(), and lowerV8F64VectorShuffle().
|
static |
Lower a vector shuffle using the SHUFPS instruction.
This is a helper routine dedicated to lowering vector shuffles using SHUFPS. It makes no assumptions about whether this is the best lowering, it simply uses it.
Definition at line 10093 of file X86ISelLowering.cpp.
References llvm::ArrayRef< T >::begin(), llvm::count_if(), llvm::find_if(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), llvm::X86ISD::SHUFP, std::swap(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16F32VectorShuffle(), lowerV16I32VectorShuffle(), lowerV4F32VectorShuffle(), lowerV8F32VectorShuffle(), and lowerV8I32VectorShuffle().
|
static |
Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
Definition at line 8856 of file X86ISelLowering.cpp.
References llvm::SmallBitVector::all(), assert(), llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MipsISD::Hi, i, llvm::MVT::i8, llvm::X86ISD::INSERTQI, isSequentialOrUndefInRange(), isUndefInRange(), llvm::ArrayRef< T >::size(), and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerV16I8VectorShuffle(), and lowerV8I16VectorShuffle().
|
static |
Lower shuffles where an entire half of a 256-bit vector is UNDEF.
This allows for fast cases such as subvector extraction/insertion or shuffling smaller vector types which can lower more efficiently.
Definition at line 11788 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::MipsISD::Hi, i, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), isSequentialOrUndefInRange(), isUndefInRange(), Offset, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v4f64, llvm::MVT::v4i64, llvm::MVT::v8f32, and llvm::MVT::v8i32.
Referenced by lower256BitVectorShuffle().
|
static |
Definition at line 8220 of file X86ISelLowering.cpp.
References llvm::ShuffleVectorSDNode::commuteMask(), createUnpackShuffleMask(), llvm::SelectionDAG::getNode(), isShuffleEquivalent(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by lowerV16F32VectorShuffle(), lowerV16I16VectorShuffle(), lowerV16I32VectorShuffle(), lowerV16I8VectorShuffle(), lowerV2F64VectorShuffle(), lowerV2I64VectorShuffle(), lowerV32I16VectorShuffle(), lowerV32I8VectorShuffle(), lowerV4F32VectorShuffle(), lowerV4F64VectorShuffle(), lowerV4I32VectorShuffle(), lowerV4I64VectorShuffle(), lowerV64I8VectorShuffle(), lowerV8F32VectorShuffle(), lowerV8F64VectorShuffle(), lowerV8I16VectorShuffle(), lowerV8I32VectorShuffle(), and lowerV8I64VectorShuffle().
|
static |
Try to lower a VSELECT instruction to a vector shuffle.
Definition at line 13444 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), i, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isNullConstant(), and llvm::BitmaskEnumDetail::Mask().
|
static |
Definition at line 16679 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), ChangeVSETULTtoVSETULE(), llvm::X86ISD::CMPM, llvm::X86ISD::CMPP, EQ, llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::APInt::getSignBit(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::AArch64CC::GT, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE42(), llvm::X86Subtarget::hasVLX(), llvm::X86Subtarget::hasXOP(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), llvm::ISD::isUnsignedIntSetCC(), llvm_unreachable, Lower256IntVSETCC(), LowerBoolVSETCC_AVX512(), LowerIntVSETCC_AVX512(), llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, SB, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::X86ISD::SUBUS, std::swap(), translateX86FSETCC(), llvm::ISD::TRUNCATE, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::X86ISD::VPCOM, llvm::X86ISD::VPCOMU, llvm::ISD::XOR, and Zero.
Referenced by combineSetCC().
|
static |
Definition at line 30928 of file X86ISelLowering.cpp.
References llvm::ISD::CTLZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::i32, llvm::Log2_32(), and llvm::ISD::SRL.
Referenced by combineOrCmpEqZeroToCtlzSrl().
|
static |
Definition at line 32580 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::X86ISD::FAND, llvm::X86ISD::FANDN, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i64, llvm::MVT::isVector(), llvm_unreachable, llvm::ISD::OR, and llvm::ISD::XOR.
Referenced by combineFAnd(), combineFAndn(), and combineFOr().
|
static |
Definition at line 22048 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADD, llvm::X86::COND_B, llvm::X86::COND_O, llvm::X86ISD::DEC, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), getSETCC(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::getVTList(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i8, llvm::X86ISD::INC, llvm::isOneConstant(), llvm_unreachable, llvm::ISD::MERGE_VALUES, llvm::ISD::SADDO, llvm::X86ISD::SMUL, llvm::X86ISD::SMUL8, llvm::ISD::SMULO, llvm::ISD::SSUBO, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, llvm::ISD::UADDO, llvm::X86ISD::UMUL, llvm::X86ISD::UMUL8, llvm::ISD::UMULO, and llvm::ISD::USUBO.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 15331 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasFp256(), llvm::MVT::i1, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerAVXExtend(), and LowerZERO_EXTEND_AVX512().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 15291 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::APInt::getNullValue(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::MVT::is512BitVector(), llvm::ISD::VSELECT, llvm::X86ISD::VTRUNC, and llvm::X86ISD::VZEXT.
Referenced by LowerZERO_EXTEND().
|
static |
Definition at line 19838 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::WinEHFuncInfo::EHGuardFrameIndex, llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getOperand(), llvm::MachineFunction::getWinEHFuncInfo(), and llvm::report_fatal_error().
Referenced by LowerINTRINSIC_W_CHAIN().
|
static |
Definition at line 19820 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::WinEHFuncInfo::EHRegNodeFrameIndex, llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getOperand(), llvm::MachineFunction::getWinEHFuncInfo(), and llvm::report_fatal_error().
Referenced by LowerINTRINSIC_W_CHAIN().
Definition at line 34036 of file X86ISelLowering.cpp.
References llvm::StringRef::empty(), llvm::StringRef::find_first_not_of(), llvm::StringRef::startswith(), and llvm::StringRef::substr().
Referenced by llvm::X86TargetLowering::ExpandInlineAsm().
|
static |
Definition at line 26587 of file X86ISelLowering.cpp.
References llvm::SmallBitVector::any(), llvm::X86ISD::BLENDI, llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::MipsISD::Hi, i, llvm::MVT::i8, llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), isRepeatedTargetShuffleMask(), isSequentialOrUndefOrZeroInRange(), isUndefInRange(), isUndefOrInRange(), isUndefOrZeroInRange(), llvm::MipsISD::Lo, matchVectorShuffleAsByteRotate(), matchVectorShuffleAsInsertPS(), matchVectorShuffleWithSHUFPD(), Offset, llvm::X86ISD::PALIGNR, llvm::X86ISD::SHUFP, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::MVT::v16f32, llvm::NVPTX::PTXLdStInstCode::V2, llvm::MVT::v2f64, llvm::MVT::v2i64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8f32, llvm::MVT::v8f64, llvm::MVT::v8i16, and llvm::MVT::v8i32.
Referenced by combineX86ShuffleChain().
|
static |
Definition at line 26489 of file X86ISelLowering.cpp.
References llvm::ShuffleVectorSDNode::commuteMask(), createUnpackShuffleMask(), llvm::MVT::getScalarSizeInBits(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isTargetShuffleEquivalent(), llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, std::swap(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::MVT::v4f32, llvm::MVT::v4f64, and llvm::MVT::v8f32.
Referenced by combineX86ShuffleChain().
|
static |
Definition at line 28452 of file X86ISelLowering.cpp.
References llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), i, llvm::isNullConstant(), and llvm::Log2_32().
Referenced by combineBasicSADPattern().
|
static |
Return true if the given stack call argument is already available in the same position (relatively) of the caller's incoming argument stack.
Definition at line 3778 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::AssertZext, llvm::ISD::BITCAST, llvm::ISD::CopyFromReg, llvm::tgtok::Def, llvm::dyn_cast(), llvm::ISD::FrameIndex, llvm::ISD::ArgFlagsTy::getByValSize(), llvm::MachineOperand::getIndex(), llvm::FrameIndexSDNode::getIndex(), llvm::CCValAssign::getLocVT(), llvm::MachineFrameInfo::getObjectOffset(), llvm::MachineFrameInfo::getObjectSize(), llvm::SDValue::getOpcode(), llvm::MachineInstr::getOpcode(), llvm::SDValue::getOperand(), llvm::MachineInstr::getOperand(), getReg(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MachineRegisterInfo::getVRegDef(), llvm::ISD::ArgFlagsTy::isByVal(), llvm::MachineOperand::isFI(), llvm::MachineFrameInfo::isFixedObjectIndex(), llvm::X86InstrInfo::isLoadFromStackSlot(), llvm::MachineFrameInfo::isObjectSExt(), llvm::MachineFrameInfo::isObjectZExt(), llvm::ISD::ArgFlagsTy::isSExt(), llvm::TargetRegisterInfo::isVirtualRegister(), llvm::ISD::ArgFlagsTy::isZExt(), Ptr, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 26345 of file X86ISelLowering.cpp.
References llvm::all_of(), assert(), llvm::ArrayRef< T >::data(), llvm::MVT::f32, llvm::MVT::f64, llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), getV4X86ShuffleImm(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), i, llvm::MVT::i32, is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is128BitVector(), is256BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isSequentialOrUndefInRange(), isUndefOrInRange(), isUndefOrZero(), llvm::BitmaskEnumDetail::Mask(), matchVectorShuffleAsShift(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, scaleShuffleMask(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::MVT::v4f64, llvm::MVT::v4i64, llvm::MVT::v8f64, llvm::MVT::v8i64, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by combineX86ShuffleChain().
|
static |
Definition at line 26230 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), i, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isTargetShuffleEquivalent(), isUndefOrEqual(), isUndefOrZeroInRange(), llvm::ScaledNumbers::MaxScale, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::ArrayRef< T >::size(), llvm::MVT::v16f32, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v8f32, llvm::MVT::v8f64, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VZEXT, and llvm::X86ISD::VZEXT_MOVL.
Referenced by combineX86ShuffleChain().
|
static |
Try to lower a vector shuffle as a byte rotation.
SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary byte-rotation of the concatenation of two vectors; pre-SSSE3 can use a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will try to generically lower a vector shuffle through such an pattern. It does not check for the profitability of lowering either as PALIGNR or PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form. This matches shuffle vectors that look like:
v8i16 [11, 12, 13, 14, 15, 0, 1, 2]
Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.
Definition at line 8635 of file X86ISelLowering.cpp.
References llvm::any_of(), is128BitLaneRepeatedShuffleMask(), matchVectorShuffleAsRotate(), llvm::SmallVectorTemplateCommon< T >::size(), and llvm::SM_SentinelZero.
Referenced by lowerVectorShuffleAsByteRotate(), and matchBinaryPermuteVectorShuffle().
|
static |
Definition at line 9654 of file X86ISelLowering.cpp.
References assert(), llvm::ArrayRef< T >::begin(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::ArrayRef< T >::end(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), i, llvm::MVT::is128BitVector(), llvm::ArrayRef< T >::size(), llvm::MVT::v4f32, and llvm::codeview::VB.
Referenced by lowerVectorShuffleAsInsertPS(), and matchBinaryPermuteVectorShuffle().
Try to lower a vector shuffle as a rotation.
This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.
Definition at line 8550 of file X86ISelLowering.cpp.
References assert(), llvm::MipsISD::Hi, i, llvm::MipsISD::Lo, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by lowerVectorShuffleAsRotate(), and matchVectorShuffleAsByteRotate().
|
static |
Try to lower a vector shuffle as a bit shift (shifts in zeros).
Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function matches elements from one of the input vectors shuffled to the left or right with zeroable elements 'shifted in'. It handles both the strictly bit-wise element shifts and the byte shift across an entire 128-bit double quad word lane.
PSHL : (little-endian) left bit shift. [ zz, 0, zz, 2 ] [ -1, 4, zz, -1 ] PSRL : (little-endian) right bit shift. [ 1, zz, 3, zz] [ -1, -1, 7, zz] PSLLDQ : (little-endian) left byte shift [ zz, 0, 1, 2, 3, 4, 5, 6] [ zz, zz, -1, -1, 2, 3, 4, -1] [ zz, zz, zz, zz, zz, zz, -1, 1] PSRLDQ : (little-endian) right byte shift [ 5, 6, 7, zz, zz, zz, zz, zz] [ -1, 5, 6, 7, zz, zz, zz, zz] [ 1, 2, -1, -1, -1, -1, zz, zz]
Definition at line 8757 of file X86ISelLowering.cpp.
References llvm::MVT::getIntegerVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasBWI(), i, llvm::MVT::i8, isSequentialOrUndefInRange(), llvm::Left, llvm::ArrayRef< T >::size(), llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VSRLI.
Referenced by lowerVectorShuffleAsShift(), and matchUnaryPermuteVectorShuffle().
|
static |
Definition at line 12090 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::f64, llvm::MVT::getScalarType(), llvm::MVT::getVectorNumElements(), i, llvm::SM_SentinelUndef, and std::swap().
Referenced by lowerVectorShuffleWithSHUFPD(), and matchBinaryPermuteVectorShuffle().
|
static |
Definition at line 33299 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::X86::COND_B, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::i1, llvm::MVT::i8, llvm::X86ISD::SETCC_CARRY, and llvm::ISD::TRUNCATE.
Referenced by combineX86SetCC().
|
static |
Create a vector constant without a load.
SSE/AVX provide the bare minimum functionality to do this, so it's all zeros, all ones, or some derivation that is cheap to calculate.
Definition at line 7327 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), getOnesVector(), llvm::SDValue::getSimpleValueType(), getZeroVector(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::v16i32, llvm::MVT::v4i32, and llvm::MVT::v8i32.
Definition at line 4069 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::hasOneUse(), llvm::ISD::isNormalStore(), and llvm::SDNode::use_begin().
Referenced by llvm::X86TargetLowering::IsDesirableToPromoteOp().
Definition at line 4073 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::hasOneUse(), llvm::SDNode::use_begin(), and llvm::ISD::ZERO_EXTEND.
Definition at line 4065 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::hasOneUse(), and llvm::ISD::isNormalLoad().
Referenced by llvm::X86TargetLowering::IsDesirableToPromoteOp(), and lowerVectorShuffleAsBroadcast().
|
static |
Return true if we might ever do TCO for calls with this calling convention.
Definition at line 2623 of file X86ISelLowering.cpp.
References llvm::CallingConv::C, canGuaranteeTCO(), llvm::CallingConv::X86_64_SysV, llvm::CallingConv::X86_64_Win64, llvm::CallingConv::X86_FastCall, llvm::CallingConv::X86_StdCall, llvm::CallingConv::X86_ThisCall, and llvm::CallingConv::X86_VectorCall.
|
static |
fold (add Y, (sete X, 0)) -> adc 0, Y (add Y, (setne X, 0)) -> sbb -1, Y (sub (sete X, 0), Y) -> sbb 0, Y (sub (setne X, 0), Y) -> adc -1, Y
Definition at line 33545 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::X86ISD::CMP, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::MVT::i32, llvm::EVT::isInteger(), llvm::X86::isZeroNode(), llvm::X86ISD::SBB, llvm::X86ISD::SETCC, llvm::ISD::SUB, and llvm::ISD::ZERO_EXTEND.
Referenced by combineAdd(), and combineSub().
|
static |
Breaks v64i1 value into two registers and adds the new node to the DAG.
Definition at line 2128 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_ELEMENT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::CCValAssign::getLocReg(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::X86Subtarget::hasBMI(), llvm::X86Subtarget::hasBWI(), llvm::MipsISD::Hi, llvm::MVT::i32, llvm::MVT::i64, llvm::X86Subtarget::is32Bit(), llvm::CCValAssign::isRegLoc(), and llvm::MipsISD::Lo.
Definition at line 5168 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), and llvm::SDValue::getOperand().
Referenced by combineANDXORWithAllOnesIntoANDNP(), combineLogicBlendIntoPBLENDV(), combinePCMPAnd1(), combineTargetShuffle(), combineVSZext(), combineX86ShuffleChain(), combineX86ShufflesRecursively(), computeZeroableShuffleElements(), EltsFromConsecutiveLoads(), getScalarValueForVectorElement(), getTargetConstantBitsFromNode(), getTargetConstantFromNode(), getTargetShuffleMaskIndices(), isFNEG(), isShuffleFoldableLoad(), LowerINSERT_SUBVECTOR(), lowerV2I64VectorShuffle(), lowerVectorShuffleAsBroadcast(), setTargetShuffleZeroElements(), and splitAndLowerVectorShuffle().
Definition at line 5174 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), and llvm::SDValue::hasOneUse().
Referenced by combineTargetShuffle(), combineTruncatedArithmetic(), combineX86ShufflesRecursively(), and LowerINSERT_SUBVECTOR().
|
static |
Returns a vector of 0s if the node in input is a vector logical shift by a constant amount which is known to be bigger than or equal to the vector element size in bits.
Definition at line 30352 of file X86ISelLowering.cpp.
References llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SDNode::getValueType(), getZeroVector(), llvm::X86Subtarget::hasInt256(), llvm::APInt::trunc(), llvm::APInt::uge(), llvm::MVT::v16i16, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v4i64, llvm::MVT::v8i16, and llvm::MVT::v8i32.
Referenced by combineShift().
|
static |
sext(add_nsw(x, C)) –> add(sext(x), C_sext) zext(add_nuw(x, C)) –> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode.
This can eliminate extend, add, and shift instructions.
Definition at line 32865 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::dyn_cast(), fuzzer::Flags, llvm::SelectionDAG::getConstant(), llvm::SDNode::getFlags(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDNodeFlags::hasNoSignedWrap(), llvm::SDNodeFlags::hasNoUnsignedWrap(), llvm::MVT::i64, llvm::SDNodeFlags::setNoSignedWrap(), llvm::SDNodeFlags::setNoUnsignedWrap(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SDNode::uses(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineSext(), and combineZext().
|
static |
When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP.
Here's the math: RegNodeBase = EntryEBP - RegNodeSize ParentFP = RegNodeBase - ParentFrameOffset Subtracting RegNodeSize takes us to the offset of the registration node, and subtracting the offset (negative on x86) takes us back to the parent FP.
Definition at line 18747 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SelectionDAG::getConstant(), llvm::MachineModuleInfo::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMCSymbol(), llvm::MachineFunction::getMMI(), llvm::Value::getName(), llvm::SelectionDAG::getNode(), llvm::MCContext::getOrCreateParentFrameOffsetSymbol(), llvm::TargetLoweringBase::getPointerTy(), llvm::GlobalValue::getRealLinkageName(), getSEHRegistrationNodeSize(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasPersonalityFn(), llvm::X86Subtarget::is64Bit(), llvm::ISD::LOCAL_RECOVER, and llvm::ISD::SUB.
Referenced by LowerINTRINSIC_WO_CHAIN().
|
static |
If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert.
Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.
Definition at line 31508 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::MemSDNode::getPointerInfo(), llvm::MaskedLoadSDNode::getSrc0(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_VECTOR_ELT, and llvm::SPII::Load.
Referenced by combineMaskedLoad().
|
static |
If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store.
Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.
Definition at line 31675 of file X86ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::MemSDNode::getPointerInfo(), llvm::SelectionDAG::getStore(), llvm::MaskedStoreSDNode::getValue(), llvm::SDValue::getValueType(), and llvm::EVT::getVectorElementType().
Referenced by combineMaskedStore().
|
static |
When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code.
Two typical patterns are handled: Pattern1: %2 = sext/zext <N x="" i8>=""> %1 to <N x="" i32>=""> %4 = sext/zext <N x="" i8>=""> %3 to <N x="" i32>=""> %5 = mul <N x="" i32>=""> %2, %4
Pattern2: %2 = zext/sext <N x="" i16>=""> %1 to <N x="" i32>=""> %4 = zext/sext <N x="" i16>=""> %3 to <N x="" i32>=""> or %4 = build_vector <N x="" i32>=""> C1, ..., CN (C1..CN are constants) %5 = mul <N x="" i32>=""> %2, %4
There are four mul shrinking modes: If %2 == sext32(trunc8(%2)), i.e., the scalar value range of %2 is -128 to 128, and the scalar value range of %4 is also -128 to 128, generate pmullw+sext32 for it (MULS8 mode). If %2 == zext32(trunc8(%2)), i.e., the scalar value range of %2 is 0 to 255, and the scalar value range of %4 is also 0 to 255, generate pmullw+zext32 for it (MULU8 mode). If %2 == sext32(trunc16(%2)), i.e., the scalar value range of %2 is -32768 to 32767, and the scalar value range of %4 is also -32768 to 32767, generate pmullw+pmulhw for it (MULS16 mode). If %2 == zext32(trunc16(%2)), i.e., the scalar value range of %2 is 0 to 65535, and the scalar value range of %4 is also 0 to 65535, generate pmullw+pmulhuw for it (MULU16 mode).
Definition at line 30027 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, canReduceVMulWidth(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), i, llvm::MVT::i16, llvm::MVT::i32, llvm::X86Subtarget::isPMULLDSlow(), Mode, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, MULS16, MULS8, MULU8, llvm::Function::optForMinSize(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::TRUNCATE, llvm::X86ISD::UNPCKL, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by combineMul().
|
static |
Calls setTargetShuffleZeroElements to resolve a target shuffle mask's inputs and set the SM_SentinelUndef and SM_SentinelZero values.
Then check the remaining input indices in case we now have a unary shuffle and adjust the Op0/Op1 inputs accordingly. Returns true if the target shuffle mask was decoded.
Definition at line 5818 of file X86ISelLowering.cpp.
References llvm::any_of(), getFauxShuffleMask(), setTargetShuffleZeroElements(), and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by combineX86ShufflesRecursively().
|
static |
Helper function to scale a shuffle or target shuffle mask, replacing each mask index with the scaled sequential indices for an equivalent narrowed mask.
This is the reverse process to canWidenShuffleElements, but can always succeed.
Definition at line 4607 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::assign(), i, and llvm::ArrayRef< T >::size().
Referenced by combineX86ShuffleChain(), lowerV4I64VectorShuffle(), lowerV8I64VectorShuffle(), and matchUnaryPermuteVectorShuffle().
|
static |
Check a target shuffle mask's inputs to see if we can set any values to SM_SentinelZero - this is for elements that are known to be zero (not just zeroable) from their inputs.
Returns true if the target shuffle mask was decoded.
Definition at line 5655 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), getTargetShuffleMask(), llvm::MVT::getVectorNumElements(), i, isTargetShuffle(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), llvm::BitmaskEnumDetail::Mask(), peekThroughBitcasts(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, and llvm::NVPTX::PTXLdStInstCode::V2.
Referenced by combineTargetShuffle(), and resolveTargetShuffleInputs().
|
static |
Return true if the function is being made into a tailcall target by changing its ABI.
Definition at line 2642 of file X86ISelLowering.cpp.
References canGuaranteeTCO().
Referenced by llvm::X86::isCalleePop().
|
static |
Generic routine to split vector shuffle into half-sized shuffles.
This routine just extracts two subvectors, shuffles them independently, and then concatenates them back together. This should work effectively with all AVX vector shuffle types.
Definition at line 11395 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MipsISD::Hi, i, llvm::MipsISD::Lo, peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T, isPodLike< T >::value >::push_back(), llvm::ArrayRef< T >::size(), and llvm::ArrayRef< T >::slice().
Referenced by lower256BitVectorShuffle(), lowerV64I8VectorShuffle(), lowerVectorShuffleAsLanePermuteAndBlend(), and lowerVectorShuffleAsSplitOrBlend().
| STATISTIC | ( | NumTailCalls | , |
| "Number of tail calls" | |||
| ) |
|
static |
Definition at line 21287 of file X86ISelLowering.cpp.
References SupportedVectorShiftWithImm().
Referenced by LowerScalarVariableShift().
|
static |
Definition at line 21267 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasVLX(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::SRA, llvm::MVT::v2i64, and llvm::MVT::v4i64.
Referenced by LowerScalarImmediateShift(), and SupportedVectorShiftWithBaseAmnt().
|
static |
Definition at line 21294 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasVLX(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::SRA, llvm::MVT::v2i64, and llvm::MVT::v4i64.
Referenced by LowerShift().
|
static |
Definition at line 4227 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, and llvm::ISD::SETULT.
Referenced by TranslateX86CC().
|
static |
Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make.
Definition at line 4246 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_INVALID, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_NS, llvm::X86::COND_P, llvm::X86::COND_S, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::ISD::getSetCCSwappedOperands(), llvm::SDValue::getValueType(), llvm::ISD::isNON_EXTLoad(), llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, std::swap(), and TranslateIntegerX86CC().
|
static |
Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs.
Definition at line 16494 of file X86ISelLowering.cpp.
References LLVM_FALLTHROUGH, llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, and std::swap().
Referenced by LowerVSETCC().
|
static |
Helper to recursively truncate vector elements in half with PACKSS.
It makes use of the fact that vector comparison results will be all-zeros or all-ones to use (vXi8 PACKSS(vYi16, vYi16)) instead of matching types. AVX2 (Int256) sub-targets require extra shuffling as the PACKSS operates within each 128-bit lane.
Definition at line 15355 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::MipsISD::Hi, llvm::tgtok::In, llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::MipsISD::Lo, llvm::X86ISD::PACKSS, llvm::MVT::v16i16, llvm::MVT::v16i8, llvm::MVT::v32i8, llvm::MVT::v4i64, and llvm::MVT::v8i16.
Referenced by combineVectorSignBitsTruncation().
|
static |
Definition at line 30586 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSplatBuildVector(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::TargetLoweringBase::isOperationLegalOrPromote(), llvm_unreachable, llvm::ISD::OR, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::TRUNCATE, llvm::ISD::XOR, llvm::ISD::ZERO_EXTEND, and llvm::APInt::zext().
Referenced by combineSext(), and combineZext().
|
static |
Check if a vector extract from a target-specific shuffle of a load can be folded into a single element load.
Similar handling for VECTOR_SHUFFLE is performed by DAGCombiner, but shuffles have been custom lowered so we need to handle those here.
Definition at line 28292 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::DataLayout::getABITypeAlignment(), llvm::MemSDNode::getAlignment(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::EVT::getTypeForEVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), isTargetShuffle(), llvm::EVT::isVector(), llvm::MemSDNode::isVolatile(), llvm::ISD::LOAD, llvm::SmallVectorTemplateCommon< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineExtractVectorElt().
1.8.6