LLVM 20.0.0git
|
#include "X86ISelLowering.h"
#include "MCTargetDesc/X86ShuffleDecode.h"
#include "X86.h"
#include "X86CallingConv.h"
#include "X86FrameLowering.h"
#include "X86InstrBuilder.h"
#include "X86IntrinsicsInfo.h"
#include "X86MachineFunctionInfo.h"
#include "X86TargetMachine.h"
#include "X86TargetObjectFile.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/ObjCARCUtil.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/IntrinsicLowering.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/EHPersonalities.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <bitset>
#include <cctype>
#include <numeric>
Go to the source code of this file.
Namespaces | |
namespace | llvm |
This is an optimization pass for GlobalISel generic memory operations. | |
namespace | llvm::X86 |
Define some predicates that are used for node matching. | |
Macros | |
#define | DEBUG_TYPE "x86-isel" |
#define | NODE_NAME_CASE(NODE) case X86ISD::NODE: return "X86ISD::" #NODE; |
#define | GET_EGPR_IF_ENABLED(OPC) (Subtarget.hasEGPR() ? OPC##_EVEX : OPC) |
Enumerations | |
enum | BitTestKind : unsigned { UndefBit , ConstantBit , NotConstantBit , ShiftBit , NotShiftBit } |
enum | { llvm::X86::MaxShuffleCombineDepth = 8 } |
enum class | ShrinkMode { MULS8 , MULU8 , MULS16 , MULU16 } |
Different mul shrinking modes. More... | |
Functions | |
static bool | isLogicOp (unsigned Opcode) |
static bool | isTargetShuffle (unsigned Opcode) |
static bool | isTargetShuffleVariableMask (unsigned Opcode) |
static bool | isX86CCSigned (unsigned X86CC) |
Return true if the condition is an signed comparison operation. | |
static X86::CondCode | TranslateIntegerX86CC (ISD::CondCode SetCCOpcode) |
static X86::CondCode | TranslateX86CC (ISD::CondCode SetCCOpcode, const SDLoc &DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG) |
Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make. | |
static bool | hasFPCMov (unsigned X86CC) |
Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. | |
static bool | useVPTERNLOG (const X86Subtarget &Subtarget, MVT VT) |
static bool | isUndefOrEqual (int Val, int CmpVal) |
Val is the undef sentinel value or equal to the specified value. | |
static bool | isUndefOrEqual (ArrayRef< int > Mask, int CmpVal) |
Return true if every element in Mask is the undef sentinel value or equal to the specified value. | |
static bool | isUndefOrEqualInRange (ArrayRef< int > Mask, int CmpVal, unsigned Pos, unsigned Size) |
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value or equal to the specified value. | |
static bool | isUndefOrZero (int Val) |
Val is either the undef or zero sentinel value. | |
static bool | isUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size) |
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value. | |
static bool | isUndefLowerHalf (ArrayRef< int > Mask) |
Return true if the mask creates a vector whose lower half is undefined. | |
static bool | isUndefUpperHalf (ArrayRef< int > Mask) |
Return true if the mask creates a vector whose upper half is undefined. | |
static bool | isInRange (int Val, int Low, int Hi) |
Return true if Val falls within the specified range (L, H]. | |
static bool | isAnyInRange (ArrayRef< int > Mask, int Low, int Hi) |
Return true if the value of any element in Mask falls within the specified range (L, H]. | |
static bool | isAnyZero (ArrayRef< int > Mask) |
Return true if the value of any element in Mask is the zero sentinel value. | |
static bool | isUndefOrInRange (int Val, int Low, int Hi) |
Return true if Val is undef or if its value falls within the specified range (L, H]. | |
static bool | isUndefOrInRange (ArrayRef< int > Mask, int Low, int Hi) |
Return true if every element in Mask is undef or if its value falls within the specified range (L, H]. | |
static bool | isUndefOrZeroOrInRange (int Val, int Low, int Hi) |
Return true if Val is undef, zero or if its value falls within the specified range (L, H]. | |
static bool | isUndefOrZeroOrInRange (ArrayRef< int > Mask, int Low, int Hi) |
Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H]. | |
static LLVM_ATTRIBUTE_UNUSED bool | isBlendOrUndef (ArrayRef< int > Mask) |
Return true if every element in Mask, is an in-place blend/select mask or is undef. | |
static bool | isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1) |
Return true if every element in Mask, beginning from position Pos and ending in Pos + Size, falls within the specified sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef. | |
static bool | isSequentialOrUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1) |
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero. | |
static bool | isUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size) |
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero. | |
static bool | isCompletePermute (ArrayRef< int > Mask) |
Return true if every element of a single input is referenced by the shuffle mask. | |
static bool | canWidenShuffleElements (ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask) |
Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled. | |
static bool | canWidenShuffleElements (ArrayRef< int > Mask, const APInt &Zeroable, bool V2IsZero, SmallVectorImpl< int > &WidenedMask) |
static bool | canWidenShuffleElements (ArrayRef< int > Mask) |
static bool | scaleShuffleElements (ArrayRef< int > Mask, unsigned NumDstElts, SmallVectorImpl< int > &ScaledMask) |
static bool | canScaleShuffleElements (ArrayRef< int > Mask, unsigned NumDstElts) |
static SDValue | getConstVector (ArrayRef< int > Values, MVT VT, SelectionDAG &DAG, const SDLoc &dl, bool IsMask=false) |
static SDValue | getConstVector (ArrayRef< APInt > Bits, const APInt &Undefs, MVT VT, SelectionDAG &DAG, const SDLoc &dl) |
static SDValue | getConstVector (ArrayRef< APInt > Bits, MVT VT, SelectionDAG &DAG, const SDLoc &dl) |
static SDValue | getZeroVector (MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl) |
Returns a vector of specified type with all zero elements. | |
static SDValue | getSplitVectorSrc (SDValue LHS, SDValue RHS, bool AllowCommute) |
static SDValue | extractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth) |
static SDValue | extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl) |
Generate a DAG to grab 128-bits from a vector > 128 bits. | |
static SDValue | extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl) |
Generate a DAG to grab 256-bits from a 512-bit vector. | |
static SDValue | insertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth) |
static SDValue | insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl) |
Generate a DAG to put 128-bits into a vector > 128 bits. | |
static SDValue | widenSubVector (MVT VT, SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl) |
Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef. | |
static SDValue | widenSubVector (SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl, unsigned WideSizeInBits) |
Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef. | |
static MVT | widenMaskVectorType (MVT VT, const X86Subtarget &Subtarget) |
Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types. | |
static SDValue | widenMaskVector (SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl) |
Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types. | |
static bool | collectConcatOps (SDNode *N, SmallVectorImpl< SDValue > &Ops, SelectionDAG &DAG) |
static SDValue | isUpperSubvectorUndef (SDValue V, const SDLoc &DL, SelectionDAG &DAG) |
static bool | isFreeToSplitVector (SDNode *N, SelectionDAG &DAG) |
static std::pair< SDValue, SDValue > | splitVector (SDValue Op, SelectionDAG &DAG, const SDLoc &dl) |
static SDValue | splitVectorOp (SDValue Op, SelectionDAG &DAG, const SDLoc &dl) |
Break an operation into 2 half sized ops and then concatenate the results. | |
static SDValue | splitVectorIntUnary (SDValue Op, SelectionDAG &DAG, const SDLoc &dl) |
Break an unary integer operation into 2 half sized ops and then concatenate the result back. | |
static SDValue | splitVectorIntBinary (SDValue Op, SelectionDAG &DAG, const SDLoc &dl) |
Break a binary integer operation into 2 half sized ops and then concatenate the result back. | |
template<typename F > | |
SDValue | SplitOpsAndApply (SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, F Builder, bool CheckBWI=true) |
static SDValue | getAVX512Node (unsigned Opcode, const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | insert1BitVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Insert i1-subvector to i1-vector. | |
static SDValue | concatSubVectors (SDValue V1, SDValue V2, SelectionDAG &DAG, const SDLoc &dl) |
static SDValue | getOnesVector (EVT VT, SelectionDAG &DAG, const SDLoc &dl) |
Returns a vector of specified type with all bits set. | |
static SDValue | getEXTEND_VECTOR_INREG (unsigned Opcode, const SDLoc &DL, EVT VT, SDValue In, SelectionDAG &DAG) |
static SDValue | getBitSelect (const SDLoc &DL, MVT VT, SDValue LHS, SDValue RHS, SDValue Mask, SelectionDAG &DAG) |
static SDValue | getVectorShuffle (SelectionDAG &DAG, EVT VT, const SDLoc &dl, SDValue V1, SDValue V2, ArrayRef< int > Mask) |
static SDValue | getUnpackl (SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2) |
Returns a vector_shuffle node for an unpackl operation. | |
static SDValue | getUnpackh (SelectionDAG &DAG, const SDLoc &dl, EVT VT, SDValue V1, SDValue V2) |
Returns a vector_shuffle node for an unpackh operation. | |
static SDValue | getPack (SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &dl, MVT VT, SDValue LHS, SDValue RHS, bool PackHiHalf=false) |
Returns a node that packs the LHS + RHS nodes together at half width. | |
static SDValue | getShuffleVectorZeroOrUndef (SDValue V2, int Idx, bool IsZero, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Return a vector_shuffle of the specified vector of zero or undef vector. | |
static ConstantPoolSDNode * | getTargetConstantPoolFromBasePtr (SDValue Ptr) |
static const Constant * | getTargetConstantFromBasePtr (SDValue Ptr) |
static const Constant * | getTargetConstantFromNode (LoadSDNode *Load) |
static const Constant * | getTargetConstantFromNode (SDValue Op) |
static bool | getTargetConstantBitsFromNode (SDValue Op, unsigned EltSizeInBits, APInt &UndefElts, SmallVectorImpl< APInt > &EltBits, bool AllowWholeUndefs=true, bool AllowPartialUndefs=false) |
bool | llvm::X86::isConstantSplat (SDValue Op, APInt &SplatVal, bool AllowPartialUndefs=true) |
If Op is a constant whose elements are all the same constant or undefined, return true and return the constant value in SplatVal . | |
static bool | getTargetShuffleMaskIndices (SDValue MaskNode, unsigned MaskEltSizeInBits, SmallVectorImpl< uint64_t > &RawMask, APInt &UndefElts) |
static SDValue | IsNOT (SDValue V, SelectionDAG &DAG) |
static void | createPackShuffleMask (MVT VT, SmallVectorImpl< int > &Mask, bool Unary, unsigned NumStages=1) |
Create a shuffle mask that matches the PACKSS/PACKUS truncation. | |
static void | getPackDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS) |
static void | getHorizDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS) |
static bool | getTargetShuffleMask (SDValue N, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask, bool &IsUnary) |
Calculates the shuffle mask corresponding to the target-specific opcode. | |
static bool | getTargetShuffleMask (SDValue N, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask) |
static void | computeZeroableShuffleElements (ArrayRef< int > Mask, SDValue V1, SDValue V2, APInt &KnownUndef, APInt &KnownZero) |
Compute whether each element of a shuffle is zeroable. | |
static bool | getTargetShuffleAndZeroables (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, APInt &KnownUndef, APInt &KnownZero) |
Decode a target shuffle mask and inputs and see if any values are known to be undef or zero from their inputs. | |
static void | resolveTargetShuffleFromZeroables (SmallVectorImpl< int > &Mask, const APInt &KnownUndef, const APInt &KnownZero, bool ResolveKnownZeros=true) |
static void | resolveZeroablesFromTargetShuffle (const SmallVectorImpl< int > &Mask, APInt &KnownUndef, APInt &KnownZero) |
static bool | createShuffleMaskFromVSELECT (SmallVectorImpl< int > &Mask, SDValue Cond, bool IsBLENDV=false) |
static bool | getTargetShuffleInputs (SDValue Op, const APInt &DemandedElts, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts) |
static bool | getFauxShuffleMask (SDValue N, const APInt &DemandedElts, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts) |
static void | resolveTargetShuffleInputsAndMask (SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask) |
Removes unused/repeated shuffle source inputs and adjusts the shuffle mask. | |
static bool | getTargetShuffleInputs (SDValue Op, const APInt &DemandedElts, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, APInt &KnownUndef, APInt &KnownZero, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts) |
Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs and then sets the SM_SentinelUndef and SM_SentinelZero values. | |
static bool | getTargetShuffleInputs (SDValue Op, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, const SelectionDAG &DAG, unsigned Depth=0, bool ResolveKnownElts=true) |
static SDValue | getBROADCAST_LOAD (unsigned Opcode, const SDLoc &DL, EVT VT, EVT MemVT, MemSDNode *Mem, unsigned Offset, SelectionDAG &DAG) |
static SDValue | getShuffleScalarElt (SDValue Op, unsigned Index, SelectionDAG &DAG, unsigned Depth) |
Returns the scalar element that will make up the i'th element of the result of the vector shuffle. | |
static SDValue | LowerBuildVectorAsInsert (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerBuildVectorv16i8 (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Custom lower build_vector of v16i8. | |
static SDValue | LowerBuildVectorv8i16 (SDValue Op, const SDLoc &DL, const APInt &NonZeroMask, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Custom lower build_vector of v8i16. | |
static SDValue | LowerBuildVectorv4x32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Custom lower build_vector of v4i32 or v4f32. | |
static SDValue | getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &dl) |
Return a vector logical shift node. | |
static SDValue | LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, const SDLoc &dl, SelectionDAG &DAG) |
static bool | findEltLoadSrc (SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset) |
static SDValue | EltsFromConsecutiveLoads (EVT VT, ArrayRef< SDValue > Elts, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsAfterLegalize) |
Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'. | |
static SDValue | combineToConsecutiveLoads (EVT VT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsAfterLegalize) |
static Constant * | getConstantVector (MVT VT, ArrayRef< APInt > Bits, const APInt &Undefs, LLVMContext &C) |
static Constant * | getConstantVector (MVT VT, const APInt &SplatValue, unsigned SplatBitSize, LLVMContext &C) |
static bool | isFoldableUseOfShuffle (SDNode *N) |
static SDValue | lowerBuildVectorAsBroadcast (BuildVectorSDNode *BVOp, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which uses: a. | |
static int | getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx) |
For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index. | |
static SDValue | buildFromShuffleMostly (SDValue Op, const SDLoc &DL, SelectionDAG &DAG) |
static SDValue | LowerBUILD_VECTORvXbf16 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerBUILD_VECTORvXi1 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static LLVM_ATTRIBUTE_UNUSED bool | isHorizOp (unsigned Opcode) |
static bool | isHorizontalBinOpPart (const BuildVectorSDNode *N, unsigned Opcode, const SDLoc &DL, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1) |
This is a helper function of LowerToHorizontalOp(). | |
static SDValue | ExpandHorizontalBinOp (const SDValue &V0, const SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI) |
Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector. | |
static bool | isAddSubOrSubAdd (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, unsigned &NumExtracts, bool &IsSubAdd) |
Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB/SUBADD operation. | |
static bool | isFMAddSubOrFMSubAdd (const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2, unsigned ExpectedUses) |
Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBADD(Opnd0 , Opnd1 ) into FMADDSUB/FMSUBADD(x, y, Opnd1 ). | |
static SDValue | lowerToAddSubOrFMAddSub (const BuildVectorSDNode *BV, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or X86ISD::FMSUBADD node. | |
static bool | isHopBuildVector (const BuildVectorSDNode *BV, SelectionDAG &DAG, unsigned &HOpcode, SDValue &V0, SDValue &V1) |
static SDValue | getHopForBuildVector (const BuildVectorSDNode *BV, const SDLoc &DL, SelectionDAG &DAG, unsigned HOpcode, SDValue V0, SDValue V1) |
static SDValue | LowerToHorizontalOp (const BuildVectorSDNode *BV, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Lower BUILD_VECTOR to a horizontal add/sub operation if possible. | |
static SDValue | LowerShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | lowerBuildVectorToBitOp (BuildVectorSDNode *Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors. | |
static SDValue | materializeVectorConstant (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Create a vector constant without a load. | |
static SDValue | createVariablePermute (MVT VT, SDValue SrcVec, SDValue IndicesVec, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source values and a vector of extraction indices. | |
static SDValue | LowerBUILD_VECTORAsVariablePermute (SDValue V, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerAVXCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerCONCAT_VECTORSvXi1 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerCONCAT_VECTORS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static bool | isNoopShuffleMask (ArrayRef< int > Mask) |
Tiny helper function to identify a no-op mask. | |
static bool | isLaneCrossingShuffleMask (unsigned LaneSizeInBits, unsigned ScalarSizeInBits, ArrayRef< int > Mask) |
Test whether there are elements crossing LaneSizeInBits lanes in this shuffle mask. | |
static bool | is128BitLaneCrossingShuffleMask (MVT VT, ArrayRef< int > Mask) |
Test whether there are elements crossing 128-bit lanes in this shuffle mask. | |
static bool | isMultiLaneShuffleMask (unsigned LaneSizeInBits, unsigned ScalarSizeInBits, ArrayRef< int > Mask) |
Test whether elements in each LaneSizeInBits lane in this shuffle mask come from multiple lanes - this is different to isLaneCrossingShuffleMask to better support 'repeated mask + lane permute' style shuffles. | |
static bool | isRepeatedShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
Test whether a shuffle mask is equivalent within each sub-lane. | |
static bool | is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
Test whether a shuffle mask is equivalent within each 128-bit lane. | |
static bool | is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask) |
static bool | is256BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
Test whether a shuffle mask is equivalent within each 256-bit lane. | |
static bool | isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, unsigned EltSizeInBits, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
Test whether a target shuffle mask is equivalent within each sub-lane. | |
static bool | isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask) |
Test whether a target shuffle mask is equivalent within each sub-lane. | |
static bool | IsElementEquivalent (int MaskSize, SDValue Op, SDValue ExpectedOp, int Idx, int ExpectedIdx) |
Checks whether the vector elements referenced by two shuffle masks are equivalent. | |
static bool | isShuffleEquivalent (ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, SDValue V1=SDValue(), SDValue V2=SDValue()) |
Checks whether a shuffle mask is equivalent to an explicit list of arguments. | |
static bool | isTargetShuffleEquivalent (MVT VT, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, const SelectionDAG &DAG, SDValue V1=SDValue(), SDValue V2=SDValue()) |
Checks whether a target shuffle mask is equivalent to an explicit pattern. | |
static bool | isUnpackWdShuffleMask (ArrayRef< int > Mask, MVT VT, const SelectionDAG &DAG) |
static bool | is128BitUnpackShuffleMask (ArrayRef< int > Mask, const SelectionDAG &DAG) |
static bool | hasIdenticalHalvesShuffleMask (ArrayRef< int > Mask) |
Return true if a shuffle mask chooses elements identically in its top and bottom halves. | |
static unsigned | getV4X86ShuffleImm (ArrayRef< int > Mask) |
Get a 4-lane 8-bit shuffle immediate for a mask. | |
static SDValue | getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG) |
static bool | isNonZeroElementsInOrder (const APInt &Zeroable, ArrayRef< int > Mask, const EVT &VectorType, bool &IsZeroSideLeft) |
static SDValue | lowerShuffleWithPSHUFB (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower a shuffle with a single PSHUFB of V1 or V2. | |
static SDValue | getMaskNode (SDValue Mask, MVT MaskVT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl) |
Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics. | |
static SDValue | lowerShuffleToEXPAND (const SDLoc &DL, MVT VT, const APInt &Zeroable, ArrayRef< int > Mask, SDValue &V1, SDValue &V2, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static bool | matchShuffleWithUNPCK (MVT VT, SDValue &V1, SDValue &V2, unsigned &UnpackOpcode, bool IsUnary, ArrayRef< int > TargetMask, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | lowerShuffleWithUNPCK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
static SDValue | lowerShuffleWithUNPCK256 (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit) followed by unpack 256-bit. | |
static bool | matchShuffleAsVTRUNC (MVT &SrcVT, MVT &DstVT, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget) |
static SDValue | getAVX512TruncNode (const SDLoc &DL, MVT DstVT, SDValue Src, const X86Subtarget &Subtarget, SelectionDAG &DAG, bool ZeroUppers) |
static SDValue | lowerShuffleWithVPMOV (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | lowerShuffleAsVTRUNC (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static int | canLowerByDroppingElements (ArrayRef< int > Mask, bool MatchEven, bool IsSingleInput) |
Check whether a compaction lowering can be done by dropping even/odd elements and compute how many times even/odd elements must be dropped. | |
static bool | matchShuffleWithPACK (MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, unsigned &PackOpcode, ArrayRef< int > TargetMask, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned MaxStages=1) |
static SDValue | lowerShuffleWithPACK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | lowerShuffleAsBitMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to emit a bitmask instruction for a shuffle. | |
static SDValue | lowerShuffleAsBitBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
Try to emit a blend instruction for a shuffle using bit math. | |
static SDValue | getVectorMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Return (and Op , Mask ) for compare instructions or (vselect Mask , Op , PreservedSrc ) for others along with the necessary casting or extending for Mask when lowering masking intrinsics. | |
static bool | matchShuffleAsBlend (MVT VT, SDValue V1, SDValue V2, MutableArrayRef< int > Mask, const APInt &Zeroable, bool &ForceV1Zero, bool &ForceV2Zero, uint64_t &BlendMask) |
static SDValue | lowerShuffleAsBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Original, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to emit a blend instruction for a shuffle. | |
static SDValue | lowerShuffleAsBlendAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool ImmBlends=false) |
Try to lower as a blend of elements from two inputs followed by a single-input permutation. | |
static SDValue | lowerShuffleAsUNPCKAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
Try to lower as an unpack of elements from two inputs followed by a single-input permutation. | |
static SDValue | lowerShuffleAsPermuteAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction. | |
static SDValue | lowerShuffleAsByteRotateAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then permuting the elements of the result in place. | |
static bool | isBroadcastShuffleMask (ArrayRef< int > Mask) |
static bool | isNoopOrBroadcastShuffleMask (ArrayRef< int > Mask) |
static bool | isSingleElementRepeatedMask (ArrayRef< int > Mask) |
Check if the Mask consists of the same element repeated multiple times. | |
static SDValue | lowerShuffleAsDecomposedShuffleMerge (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Generic routine to decompose a shuffle and blend into independent blends and permutes. | |
static int | matchShuffleAsBitRotate (MVT &RotateVT, int EltSizeInBits, const X86Subtarget &Subtarget, ArrayRef< int > Mask) |
static SDValue | lowerShuffleAsBitRotate (const SDLoc &DL, MVT VT, SDValue V1, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Lower shuffle using X86ISD::VROTLI rotations. | |
static int | matchShuffleAsElementRotate (SDValue &V1, SDValue &V2, ArrayRef< int > Mask) |
Try to match a vector shuffle as an element rotation. | |
static int | matchShuffleAsByteRotate (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask) |
Try to lower a vector shuffle as a byte rotation. | |
static SDValue | lowerShuffleAsByteRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | lowerShuffleAsVALIGN (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower a vector shuffle as a dword/qword rotation. | |
static SDValue | lowerShuffleAsByteShiftMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower a vector shuffle as a byte shift sequence. | |
static int | matchShuffleAsShift (MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable, const X86Subtarget &Subtarget) |
Try to lower a vector shuffle as a bit shift (shifts in zeros). | |
static SDValue | lowerShuffleAsShift (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG, bool BitwiseOnly) |
static bool | matchShuffleAsEXTRQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx, const APInt &Zeroable) |
static bool | matchShuffleAsINSERTQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx) |
static SDValue | lowerShuffleWithSSE4A (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG) |
Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ. | |
static SDValue | lowerShuffleAsSpecificZeroOrAnyExtend (const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Lower a vector shuffle as a zero or any extension. | |
static SDValue | lowerShuffleAsZeroOrAnyExtend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower a vector shuffle as a zero extension on any microarch. | |
static SDValue | getScalarValueForVectorElement (SDValue V, int Idx, SelectionDAG &DAG) |
Try to get a scalar value for a specific element of a vector. | |
static bool | isShuffleFoldableLoad (SDValue V) |
Helper to test for a load that can be folded with x86 shuffles. | |
template<typename T > | |
static bool | isSoftF16 (T VT, const X86Subtarget &Subtarget) |
static SDValue | lowerShuffleAsElementInsertion (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower insertion of a single element into a zero vector. | |
static SDValue | lowerShuffleAsTruncBroadcast (const SDLoc &DL, MVT VT, SDValue V0, int BroadcastIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements. | |
static bool | isSingleSHUFPSMask (ArrayRef< int > Mask) |
Test whether this can be lowered with a single SHUFPS instruction. | |
static bool | isShuffleMaskInputInPlace (int Input, ArrayRef< int > Mask) |
Test whether the specified input (0 or 1) is in-place blended by the given mask. | |
static SDValue | lowerShuffleOfExtractsAsVperm (const SDLoc &DL, SDValue N0, SDValue N1, ArrayRef< int > Mask, SelectionDAG &DAG) |
If we are extracting two 128-bit halves of a vector and shuffling the result, match that to a 256-bit AVX2 vperm* instruction to avoid a multi-shuffle lowering. | |
static SDValue | lowerShuffleAsBroadcast (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower broadcast of a single element. | |
static bool | matchShuffleAsInsertPS (SDValue &V1, SDValue &V2, unsigned &InsertPSMask, const APInt &Zeroable, ArrayRef< int > Mask, SelectionDAG &DAG) |
static SDValue | lowerShuffleAsInsertPS (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG) |
static SDValue | lowerV2F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 2-lane 64-bit floating point shuffles. | |
static SDValue | lowerV2I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 2-lane 64-bit integer shuffles. | |
static SDValue | lowerShuffleWithSHUFPS (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG) |
Lower a vector shuffle using the SHUFPS instruction. | |
static SDValue | lowerV4F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Lower 4-lane 32-bit floating point shuffles. | |
static SDValue | lowerV4I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Lower 4-lane i32 vector shuffles. | |
static SDValue | lowerV8I16GeneralSingleInputShuffle (const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part. | |
static SDValue | lowerShuffleAsBlendOfPSHUFBs (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) |
Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used. | |
static SDValue | lowerV8I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Generic lowering of 8-lane i16 shuffles. | |
static SDValue | lowerV8F16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Lower 8-lane 16-bit floating point shuffles. | |
static SDValue | lowerShuffleWithPERMV (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | lowerV16I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Generic lowering of v16i8 shuffles. | |
static SDValue | lower128BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Dispatching routine to lower various 128-bit x86 vector shuffles. | |
static SDValue | splitAndLowerShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool SimpleOnly) |
Generic routine to split vector shuffle into half-sized shuffles. | |
static SDValue | lowerShuffleAsSplitOrBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Either split a vector in halves or decompose the shuffles and the blend/unpack. | |
static SDValue | lowerShuffleAsLanePermuteAndSHUFP (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
static SDValue | lowerShuffleAsLanePermuteAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane permutation. | |
static void | computeInLaneShuffleMask (const ArrayRef< int > &Mask, int LaneSize, SmallVector< int > &InLaneMask) |
Helper to get compute inlane shuffle mask for a complete shuffle mask. | |
static SDValue | lowerShuffleAsLanePermuteAndShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one source with a lane permutation. | |
static SDValue | lowerV2X128Shuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering 2-lane 128-bit shuffles. | |
static SDValue | lowerShuffleAsLanePermuteAndRepeatedMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane. | |
static bool | getHalfShuffleMask (ArrayRef< int > Mask, MutableArrayRef< int > HalfMask, int &HalfIdx1, int &HalfIdx2) |
If the input shuffle mask results in a vector that is undefined in all upper or lower half elements and that mask accesses only 2 halves of the shuffle's operands, return true. | |
static SDValue | getShuffleHalfVectors (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > HalfMask, int HalfIdx1, int HalfIdx2, bool UndefLower, SelectionDAG &DAG, bool UseConcat=false) |
Given the output values from getHalfShuffleMask(), create a half width shuffle of extracted vectors followed by an insert back to full width. | |
static SDValue | lowerShuffleWithUndefHalf (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF. | |
static SDValue | lowerShuffleAsRepeatedMaskAndLanePermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes. | |
static bool | matchShuffleWithSHUFPD (MVT VT, SDValue &V1, SDValue &V2, bool &ForceV1Zero, bool &ForceV2Zero, unsigned &ShuffleImm, ArrayRef< int > Mask, const APInt &Zeroable) |
static SDValue | lowerShuffleWithSHUFPD (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | lowerShuffleAsVTRUNCAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG) |
static SDValue | lowerShufflePairAsUNPCKAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG) |
static SDValue | lowerV4F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 4-lane 64-bit floating point shuffles. | |
static SDValue | lowerV4I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 4-lane 64-bit integer shuffles. | |
static SDValue | lowerV8F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 8-lane 32-bit floating point shuffles. | |
static SDValue | lowerV8I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 8-lane 32-bit integer shuffles. | |
static SDValue | lowerV16I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 16-lane 16-bit integer shuffles. | |
static SDValue | lowerV32I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 32-lane 8-bit integer shuffles. | |
static SDValue | lower256BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
High-level routine to lower various 256-bit x86 vector shuffles. | |
static SDValue | lowerV4X128Shuffle (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower a vector shuffle as a 128-bit shuffles. | |
static SDValue | lowerV8F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 8-lane 64-bit floating point shuffles. | |
static SDValue | lowerV16F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 16-lane 32-bit floating point shuffles. | |
static SDValue | lowerV8I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 8-lane 64-bit integer shuffles. | |
static SDValue | lowerV16I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 16-lane 32-bit integer shuffles. | |
static SDValue | lowerV32I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 32-lane 16-bit integer shuffles. | |
static SDValue | lowerV64I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle lowering of 64-lane 8-bit integer shuffles. | |
static SDValue | lower512BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
High-level routine to lower various 512-bit x86 vector shuffles. | |
static SDValue | lower1BitShuffleAsKSHIFTR (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static int | match1BitShuffleAsKSHIFT (unsigned &Opcode, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable) |
static SDValue | lower1BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static bool | canonicalizeShuffleMaskWithCommute (ArrayRef< int > Mask) |
Helper function that returns true if the shuffle mask should be commuted to improve canonicalization. | |
static bool | canCombineAsMaskOperation (SDValue V, const X86Subtarget &Subtarget) |
static SDValue | canonicalizeShuffleMaskWithHorizOp (MutableArrayRef< SDValue > Ops, MutableArrayRef< int > Mask, unsigned RootSizeInBits, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | lowerVECTOR_SHUFFLE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Top-level lowering for x86 vector shuffles. | |
static SDValue | lowerVSELECTtoVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to lower a VSELECT instruction to a vector shuffle. | |
static SDValue | LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG) |
static SDValue | ExtractBitFromMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Extract one bit from mask vector, like v16i1 or v8i1. | |
static APInt | getExtractedDemandedElts (SDNode *N) |
static SDValue | InsertBitToMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Insert one bit to mask vector, like v16i1 or v8i1. | |
static SDValue | LowerSCALAR_TO_VECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | GetTLSADDR (SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, SDValue *InGlue, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LocalDynamic=false) |
static SDValue | LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT) |
static SDValue | LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT) |
static SDValue | LowerToTLSGeneralDynamicModelX32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT) |
static SDValue | LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool Is64Bit, bool Is64BitLP64) |
static SDValue | LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC) |
static SDValue | LowerShiftParts (SDValue Op, SelectionDAG &DAG) |
Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount. | |
static SDValue | LowerI64IntToFP_AVX512DQ (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerI64IntToFP16 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static bool | useVectorCast (unsigned Opcode, MVT FromVT, MVT ToVT, const X86Subtarget &Subtarget) |
static SDValue | vectorizeExtractedCast (SDValue Cast, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Given a scalar cast operation that is extracted from a vector, try to vectorize the cast op followed by extraction. | |
static SDValue | lowerFPToIntToFP (SDValue CastToFP, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Given a scalar cast to FP with a cast to integer operand (almost an ftrunc), try to vectorize the cast ops. | |
static SDValue | lowerINT_TO_FP_vXi64 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | promoteXINT_TO_FP (SDValue Op, const SDLoc &dl, SelectionDAG &DAG) |
static bool | isLegalConversion (MVT VT, bool IsSigned, const X86Subtarget &Subtarget) |
static bool | shouldUseHorizontalOp (bool IsSingleSource, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Horizontal vector math instructions may be slower than normal math with shuffles. | |
static SDValue | LowerUINT_TO_FP_i64 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
64-bit unsigned integer to double expansion. | |
static SDValue | LowerUINT_TO_FP_i32 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
32-bit unsigned integer to float expansion. | |
static SDValue | lowerUINT_TO_FP_v2i32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | lowerUINT_TO_FP_vXi32 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | lowerUINT_TO_FP_vec (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerAVXExtend (SDValue Op, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | SplitAndExtendv16i1 (unsigned ExtOpc, MVT VT, SDValue In, const SDLoc &dl, SelectionDAG &DAG) |
static SDValue | LowerZERO_EXTEND_Mask (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerZERO_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | truncateVectorWithPACK (unsigned Opcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Helper to recursively truncate vector elements in half with PACKSS/PACKUS. | |
static SDValue | truncateVectorWithPACKUS (EVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Truncate using inreg zero extension (AND mask) and X86ISD::PACKUS. | |
static SDValue | truncateVectorWithPACKSS (EVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Truncate using inreg sign extension and X86ISD::PACKSS. | |
static SDValue | matchTruncateWithPACK (unsigned &PackOpcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Helper to determine if In truncated to DstVT has the necessary signbits / leading zero bits to be truncated with PACKSS / PACKUS, possibly by converting a SRL node to SRA for sign extension. | |
static SDValue | LowerTruncateVecPackWithSignBits (MVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
This function lowers a vector truncation of 'extended sign-bits' or 'extended zero-bits' values. | |
static SDValue | LowerTruncateVecPack (MVT DstVT, SDValue In, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
This function lowers a vector truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations. | |
static SDValue | LowerTruncateVecI1 (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | expandFP_TO_UINT_SSE (MVT VT, SDValue Src, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerFP16_TO_FP (SDValue Op, SelectionDAG &DAG) |
static SDValue | LowerFP_TO_FP16 (SDValue Op, SelectionDAG &DAG) |
static SDValue | lowerAddSubToHorizontalOp (SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation. | |
static SDValue | LowerFROUND (SDValue Op, SelectionDAG &DAG) |
ISD::FROUND is defined to round to nearest with ties rounding away from 0. | |
static SDValue | LowerFABSorFNEG (SDValue Op, SelectionDAG &DAG) |
The only differences between FABS and FNEG are the mask and the logic op. | |
static SDValue | LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG) |
static SDValue | LowerFGETSIGN (SDValue Op, SelectionDAG &DAG) |
static SDValue | getBT (SDValue Src, SDValue BitNo, const SDLoc &DL, SelectionDAG &DAG) |
Helper for attempting to create a X86ISD::BT node. | |
static SDValue | getSETCC (X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl, SelectionDAG &DAG) |
Helper for creating a X86ISD::SETCC node. | |
static bool | isOrXorXorTree (SDValue X, bool Root=true) |
Recursive helper for combineVectorSizedSetCCEquality() to see if we have a recognizable memcmp expansion. | |
template<typename F > | |
static SDValue | emitOrXorXorTree (SDValue X, const SDLoc &DL, SelectionDAG &DAG, EVT VecVT, EVT CmpVT, bool HasPT, F SToV) |
Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp expansion. | |
static SDValue | combineVectorSizedSetCCEquality (EVT VT, SDValue X, SDValue Y, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Try to map a 128-bit or larger integer comparison to vector instructions before type legalization splits it up into chunks. | |
static bool | matchScalarReduction (SDValue Op, ISD::NodeType BinOp, SmallVectorImpl< SDValue > &SrcOps, SmallVectorImpl< APInt > *SrcMask=nullptr) |
Helper for matching BINOP(EXTRACTELT(X,0),BINOP(EXTRACTELT(X,1),...)) style scalarized (associative) reduction patterns. | |
static SDValue | LowerVectorAllEqual (const SDLoc &DL, SDValue LHS, SDValue RHS, ISD::CondCode CC, const APInt &OriginalMask, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC) |
static SDValue | MatchVectorAllEqualTest (SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC) |
static bool | hasNonFlagsUse (SDValue Op) |
return true if Op has a use that doesn't just read flags. | |
static bool | isProfitableToUseFlagOp (SDValue Op) |
static SDValue | EmitTest (SDValue Op, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Emit nodes that will be selected as "test Op0,Op0", or something equivalent. | |
static SDValue | EmitCmp (SDValue Op0, SDValue Op1, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Emit nodes that will be selected as "cmp Op0,Op1", or something equivalent. | |
static SDValue | LowerAndToBT (SDValue And, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, X86::CondCode &X86CC) |
Result of 'and' is compared against zero. | |
static bool | cheapX86FSETCC_SSE (ISD::CondCode SetCCOpcode) |
static unsigned | translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1, bool &IsAlwaysSignaling) |
Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs. | |
static SDValue | splitIntVSETCC (EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SelectionDAG &DAG, const SDLoc &dl) |
Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back. | |
static SDValue | LowerIntVSETCC_AVX512 (SDValue Op, const SDLoc &dl, SelectionDAG &DAG) |
static SDValue | incDecVectorConstant (SDValue V, SelectionDAG &DAG, bool IsInc, bool NSW) |
Given a buildvector constant, return a new vector constant with each element incremented or decremented. | |
static SDValue | LowerVSETCCWithSUBUS (SDValue Op0, SDValue Op1, MVT VT, ISD::CondCode Cond, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
As another special case, use PSUBUS[BW] when it's profitable. | |
static SDValue | LowerVSETCC (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | EmitAVX512Test (SDValue Op0, SDValue Op1, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue &X86CC) |
static std::pair< SDValue, SDValue > | getX86XALUOOp (X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) |
static SDValue | LowerXALUO (SDValue Op, SelectionDAG &DAG) |
static bool | isX86LogicalCmp (SDValue Op) |
Return true if opcode is a X86 logical comparison. | |
static bool | isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG) |
static SDValue | LowerSIGN_EXTEND_Mask (SDValue Op, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerANY_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerEXTEND_VECTOR_INREG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerSIGN_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | splitVectorStore (StoreSDNode *Store, SelectionDAG &DAG) |
Change a vector store into a pair of half-size vector stores. | |
static SDValue | scalarizeVectorStore (StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG) |
Scalarize a vector store, bitcasting to TargetVT to determine the scalar type. | |
static SDValue | LowerStore (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerLoad (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static bool | isAndOrOfSetCCs (SDValue Op, unsigned &Opc) |
Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR. | |
static SDValue | LowerVACOPY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static unsigned | getTargetVShiftUniformOpcode (unsigned Opc, bool IsVariable) |
static SDValue | getTargetVShiftByConstNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG) |
Handle vector element shifts where the shift amount is a constant. | |
static SDValue | getTargetVShiftNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, SDValue ShAmt, int ShAmtIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Handle vector element shifts by a splat shift amount. | |
static SDValue | getScalarMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Creates an SDNode for a predicated scalar operation. | |
static int | getSEHRegistrationNodeSize (const Function *Fn) |
static SDValue | recoverFramePointer (SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP) |
When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP. | |
static SDValue | getAVX2GatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget) |
static SDValue | getGatherNode (SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget) |
static SDValue | getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget) |
static SDValue | getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget) |
static SDValue | expandIntrinsicWChainHelper (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, unsigned TargetOpcode, unsigned SrcReg, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results) |
Handles the lowering of builtin intrinsics with chain that return their value into registers EDX:EAX. | |
static void | getReadTimeStampCounter (SDNode *N, const SDLoc &DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results) |
Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp). | |
static SDValue | LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | MarkEHRegistrationNode (SDValue Op, SelectionDAG &DAG) |
static SDValue | MarkEHGuard (SDValue Op, SelectionDAG &DAG) |
static SDValue | EmitTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG) |
Emit Truncating Store with signed or unsigned saturation. | |
static SDValue | EmitMaskedTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &DL, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG) |
Emit Masked Truncating Store with signed or unsigned saturation. | |
static SDValue | LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG) |
static SDValue | createSetFPEnvNodes (SDValue Ptr, SDValue Chain, const SDLoc &DL, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerVectorCTLZ_AVX512CDI (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Lower a vector CTLZ using native supported vector CTLZ instruction. | |
static SDValue | LowerVectorCTLZInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerVectorCTLZ (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerCTLZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerCTTZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | lowerAddSub (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerADDSAT_SUBSAT (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerABS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerAVG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerMINMAX (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerFMINIMUM_FMAXIMUM (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerABD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerMUL (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowervXi8MulWithUNPCK (SDValue A, SDValue B, const SDLoc &dl, MVT VT, bool IsSigned, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue *Low=nullptr) |
static SDValue | LowerMULH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerMULO (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
uint64_t | getGFNICtrlImm (unsigned Opcode, unsigned Amt=0) |
SDValue | getGFNICtrlMask (unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, MVT VT, unsigned Amt=0) |
static bool | supportedVectorShiftWithImm (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode) |
static bool | supportedVectorShiftWithBaseAmnt (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode) |
static bool | supportedVectorVarShift (EVT VT, const X86Subtarget &Subtarget, unsigned Opcode) |
static SDValue | LowerShiftByScalarImmediate (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerShiftByScalarVariable (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | convertShiftLeftToScale (SDValue Amt, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerFunnelShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerRotate (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static std::pair< Value *, BitTestKind > | FindSingleBitChange (Value *V) |
static bool | shouldExpandCmpArithRMWInIR (AtomicRMWInst *AI) |
static SDValue | emitLockedStackOp (SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue Chain, const SDLoc &DL) |
Emit a locked operation on a stack location which does not change any memory location, but does involve a lock prefix. | |
static SDValue | LowerATOMIC_FENCE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerCMP_SWAP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | getPMOVMSKB (const SDLoc &DL, SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerBITCAST (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerHorizontalByteSum (SDValue V, MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Compute the horizontal sum of bytes in V for the elements of VT. | |
static SDValue | LowerVectorCTPOPInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerVectorCTPOP (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerCTPOP (SDValue N, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerBITREVERSE_XOP (SDValue Op, SelectionDAG &DAG) |
static SDValue | LowerBITREVERSE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerPARITY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | lowerAtomicArithWithLOCK (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | lowerAtomicArith (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Lower atomic_load_ops into LOCK-prefixed operations. | |
static SDValue | LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | LowerADDSUBO_CARRY (SDValue Op, SelectionDAG &DAG) |
static SDValue | LowerFSINCOS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | ExtendToType (SDValue InOp, MVT NVT, SelectionDAG &DAG, bool FillWithZeroes=false) |
Widen a vector input to a vector of NVT. | |
static SDValue | LowerMSCATTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerMLOAD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerMSTORE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerMGATHER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static SDValue | LowerADDRSPACECAST (SDValue Op, SelectionDAG &DAG) |
static SDValue | LowerCVTPS2PH (SDValue Op, SelectionDAG &DAG) |
static SDValue | LowerPREFETCH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
static StringRef | getInstrStrFromOpNo (const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) |
static SDValue | getFlagsOfCmpZeroFori1 (SelectionDAG &DAG, const SDLoc &DL, SDValue Mask) |
static bool | isEFLAGSLiveAfter (MachineBasicBlock::iterator Itr, MachineBasicBlock *BB) |
static MachineBasicBlock * | emitXBegin (MachineInstr &MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII) |
Utility function to emit xbegin specifying the start of an RTM region. | |
static bool | checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI) |
static bool | isCMOVPseudo (MachineInstr &MI) |
static MachineInstrBuilder | createPHIsForCMOVsInSinkBB (MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB, MachineBasicBlock *SinkMBB) |
static unsigned | getSUBriOpcode (bool IsLP64) |
static unsigned | getOpcodeForIndirectThunk (unsigned RPOpc) |
static const char * | getIndirectThunkSymbol (const X86Subtarget &Subtarget, unsigned Reg) |
static void | computeKnownBitsForPSADBW (SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) |
static void | computeKnownBitsForPMADDWD (SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) |
static void | computeKnownBitsForPMADDUBSW (SDValue LHS, SDValue RHS, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) |
static KnownBits | computeKnownBitsForHorizontalOperation (const SDValue Op, const APInt &DemandedElts, unsigned Depth, const SelectionDAG &DAG, const function_ref< KnownBits(const KnownBits &, const KnownBits &)> KnownBitsFunc) |
static SDValue | narrowLoadToVZLoad (LoadSDNode *LN, MVT MemVT, MVT VT, SelectionDAG &DAG) |
static bool | matchUnaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue V1, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT) |
static bool | matchUnaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, const SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) |
static bool | matchBinaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT, bool IsUnary) |
static bool | matchBinaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) |
static SDValue | combineX86ShuffleChainWithExtract (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineX86ShuffleChain (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Combine an arbitrary chain of shuffles into a single instruction if possible. | |
static SDValue | combineX86ShufflesConstants (MVT VT, ArrayRef< SDValue > Ops, ArrayRef< int > Mask, bool HasVariableMask, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget) |
static SDValue | combineX86ShufflesRecursively (ArrayRef< SDValue > SrcOps, int SrcOpIndex, SDValue Root, ArrayRef< int > RootMask, ArrayRef< const SDNode * > SrcNodes, unsigned Depth, unsigned MaxDepth, bool HasVariableMask, bool AllowVariableCrossLaneMask, bool AllowVariablePerLaneMask, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Fully generic combining of x86 shuffle instructions. | |
static SDValue | combineX86ShufflesRecursively (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Helper entry wrapper to combineX86ShufflesRecursively. | |
static SmallVector< int, 4 > | getPSHUFShuffleMask (SDValue N) |
Get the PSHUF-style mask from PSHUF node. | |
static SDValue | combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG) |
Search for a combinable shuffle across a chain ending in pshufd. | |
static SDValue | combineCommutableSHUFP (SDValue N, MVT VT, const SDLoc &DL, SelectionDAG &DAG) |
static SDValue | combineBlendOfPermutes (MVT VT, SDValue N0, SDValue N1, ArrayRef< int > BlendMask, const APInt &DemandedElts, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL) |
static bool | isUnaryOp (unsigned Opcode) |
static SDValue | canonicalizeShuffleWithOp (SDValue N, SelectionDAG &DAG, const SDLoc &DL) |
static SDValue | canonicalizeLaneShuffleWithRepeatedOps (SDValue V, SelectionDAG &DAG, const SDLoc &DL) |
Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()). | |
static SDValue | combineTargetShuffle (SDValue N, const SDLoc &DL, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
Try to combine x86 target specific shuffles. | |
static bool | isAddSubOrSubAddMask (ArrayRef< int > Mask, bool &Op0Even) |
Checks if the shuffle mask takes subsequent elements alternately from two vectors. | |
static bool | isAddSubOrSubAdd (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, bool &IsSubAdd) |
Returns true iff the shuffle node N can be replaced with ADDSUB(SUBADD) operation. | |
static SDValue | combineShuffleToFMAddSub (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Combine shuffle of two fma nodes into FMAddSub or FMSubAdd. | |
static SDValue | combineShuffleToAddSubOrFMAddSub (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG) |
Try to combine a shuffle into a target-specific add-sub or mul-add-sub node. | |
static SDValue | combineShuffleOfConcatUndef (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | narrowShuffle (ShuffleVectorSDNode *Shuf, SelectionDAG &DAG) |
If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the low half of each source vector and does not set any high half elements in the destination vector, narrow the shuffle to half its original size. | |
static SDValue | combineShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static bool | checkBitcastSrcVectorSize (SDValue Src, unsigned Size, bool AllowTruncate) |
static unsigned | getAltBitOpcode (unsigned Opcode) |
static SDValue | adjustBitcastSrcVectorSSE1 (SelectionDAG &DAG, SDValue Src, const SDLoc &DL) |
static SDValue | signExtendBitcastSrcVector (SelectionDAG &DAG, EVT SExtVT, SDValue Src, const SDLoc &DL) |
static SDValue | combineBitcastvxi1 (SelectionDAG &DAG, EVT VT, SDValue Src, const SDLoc &DL, const X86Subtarget &Subtarget) |
static SDValue | combinevXi1ConstantToInteger (SDValue Op, SelectionDAG &DAG) |
static SDValue | combineCastedMaskArithmetic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | createMMXBuildVector (BuildVectorSDNode *BV, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineBitcastToBoolVector (EVT VT, SDValue V, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned Depth=0) |
static SDValue | combineBitcast (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static bool | detectExtMul (SelectionDAG &DAG, const SDValue &Mul, SDValue &Op0, SDValue &Op1) |
static bool | detectZextAbsDiff (const SDValue &Abs, SDValue &Op0, SDValue &Op1) |
static SDValue | createVPDPBUSD (SelectionDAG &DAG, SDValue LHS, SDValue RHS, unsigned &LogBias, const SDLoc &DL, const X86Subtarget &Subtarget) |
static SDValue | createPSADBW (SelectionDAG &DAG, const SDValue &Zext0, const SDValue &Zext1, const SDLoc &DL, const X86Subtarget &Subtarget) |
static SDValue | combineMinMaxReduction (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combinePredicateReduction (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineVPDPBUSDPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineBasicSADPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineExtractFromVectorLoad (SDNode *N, EVT VecVT, SDValue SrcVec, uint64_t Idx, const SDLoc &dl, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineExtractWithShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | scalarizeExtEltFP (SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Extracting a scalar FP value from vector element 0 is free, so extract each operand first, then perform the math as a scalar op. | |
static SDValue | combineArithReduction (SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Try to convert a vector reduction sequence composed of binops and shuffles into horizontal ops. | |
static SDValue | combineExtractVectorElt (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence. | |
static SDValue | combineToExtendBoolVectorInReg (unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N0, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineVSelectWithAllOnesOrZeros (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation. | |
static SDValue | narrowVectorSelect (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget) |
If both arms of a vector select are concatenated vectors, split the select, and concatenate the result to eliminate a wide (256-bit) vector instruction: vselect Cond, (concat T0, T1), (concat F0, F1) --> concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1) | |
static SDValue | combineSelectOfTwoConstants (SDNode *N, SelectionDAG &DAG, const SDLoc &DL) |
static SDValue | combineVSelectToBLENDV (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element. | |
static SDValue | combineLogicBlendIntoConditionalNegate (EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | commuteSelect (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget) |
static SDValue | combineSelect (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
Do target-specific dag combines on SELECT and VSELECT nodes. | |
static SDValue | combineSetCCAtomicArith (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction. | |
static SDValue | checkSignTestSetCCCombine (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG) |
static SDValue | checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC) |
static bool | checkBoolTestAndOrSetCCCombine (SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd) |
Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS. | |
static SDValue | combineCarryThroughADD (SDValue EFLAGS, SelectionDAG &DAG) |
static SDValue | combinePTESTCC (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
If we are inverting an PTEST/TESTP operand, attempt to adjust the CC to avoid the inversion. | |
static SDValue | combineSetCCMOVMSK (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineSetCCEFLAGS (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values. | |
static SDValue | combineCMov (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]. | |
static bool | canReduceVMulWidth (SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode) |
static SDValue | reduceVMULWidth (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code. | |
static SDValue | combineMulSpecial (uint64_t MulAmt, SDNode *N, SelectionDAG &DAG, EVT VT, const SDLoc &DL) |
static SDValue | combineMulToPMADDWD (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineMulToPMULDQ (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineMul (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineShiftToPMULH (SDNode *N, SelectionDAG &DAG, const SDLoc &DL, const X86Subtarget &Subtarget) |
static SDValue | combineShiftLeft (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineShiftRightArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineShiftRightLogical (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineHorizOpWithShuffle (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineVectorPack (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineVectorHADDSUB (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineVectorShiftVar (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineVectorShiftImm (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineVectorInsert (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineCompareEqual (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends. | |
static SDValue | combineAndNotIntoANDNP (SDNode *N, SelectionDAG &DAG) |
Try to fold: (and (xor X, -1), Y) -> (andnp X, Y). | |
static SDValue | combineAndShuffleNot (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Try to fold: and (vector_shuffle<Z,...,Z> (insert_vector_elt undef, (xor X, -1), Z), undef), Y -> andnp (vector_shuffle<Z,...,Z> (insert_vector_elt undef, X, Z), undef), Y. | |
static SDValue | PromoteMaskArithmetic (SDValue N, const SDLoc &DL, EVT VT, SelectionDAG &DAG, unsigned Depth) |
static SDValue | PromoteMaskArithmetic (SDValue N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static unsigned | convertIntLogicToFPLogicOpcode (unsigned Opcode) |
static SDValue | convertIntLogicToFPLogic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
If both input operands of a logic op are being cast from floating-point types or FP compares, try to convert this into a floating-point logic node to avoid unnecessary moves from SSE to integer registers. | |
static SDValue | combineBitOpWithMOVMSK (SDNode *N, SelectionDAG &DAG) |
static SDValue | combineBitOpWithShift (SDNode *N, SelectionDAG &DAG) |
static SDValue | combineBitOpWithPACK (SDNode *N, SelectionDAG &DAG) |
static SDValue | combineAndMaskToShift (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
If this is a zero/all-bits result that is bitwise-anded with a low bits mask. | |
static SDValue | getIndexFromUnindexedLoad (LoadSDNode *Ld) |
static bool | hasBZHI (const X86Subtarget &Subtarget, MVT VT) |
static SDValue | combineAndLoadToBZHI (SDNode *Node, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineScalarAndWithMaskSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | getBMIMatchingOp (unsigned Opc, SelectionDAG &DAG, SDValue OpMustEq, SDValue Op, unsigned Depth) |
static SDValue | combineBMILogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineX86SubCmpForFlags (SDNode *N, SDValue Flag, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST) |
static SDValue | combineAndOrForCcmpCtest (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST) |
static SDValue | combineAnd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | canonicalizeBitSelect (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static bool | matchLogicBlend (SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) |
static SDValue | combineLogicBlendIntoPBLENDV (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | lowerX86CmpEqZeroToCtlzSrl (SDValue Op, SelectionDAG &DAG) |
static SDValue | combineOrCmpEqZeroToCtlzSrl (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | foldMaskedMergeImpl (SDValue And0_L, SDValue And0_R, SDValue And1_L, SDValue And1_R, const SDLoc &DL, SelectionDAG &DAG) |
static SDValue | foldMaskedMerge (SDNode *Node, SelectionDAG &DAG) |
Fold "masked merge" expressions like (m & x) | (~m & y) into the equivalent ((x ^ y) & m) ^ y) pattern. | |
static SDValue | combineAddOrSubToADCOrSBB (bool IsSub, const SDLoc &DL, EVT VT, SDValue X, SDValue Y, SelectionDAG &DAG, bool ZeroSecondOpOnly=false) |
If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB. | |
static SDValue | combineAddOrSubToADCOrSBB (SDNode *N, const SDLoc &DL, SelectionDAG &DAG) |
If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB. | |
static SDValue | combineOrXorWithSETCC (SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG) |
static SDValue | combineOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | foldXorTruncShiftIntoCmp (SDNode *N, SelectionDAG &DAG) |
Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1) | |
static SDValue | foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1. | |
static SDValue | detectUSatPattern (SDValue In, EVT VT, SelectionDAG &DAG, const SDLoc &DL) |
Detect patterns of truncation with unsigned saturation: | |
static SDValue | detectSSatPattern (SDValue In, EVT VT, bool MatchPackUS=false) |
Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type). | |
static SDValue | combineTruncateWithSat (SDValue In, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineConstantPoolLoads (SDNode *N, const SDLoc &dl, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static int | getOneTrueElt (SDValue V) |
If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element. | |
static bool | getParamsForOneTrueMaskedElt (MaskedLoadStoreSDNode *MaskedOp, SelectionDAG &DAG, SDValue &Addr, SDValue &Index, Align &Alignment, unsigned &Offset) |
Given a masked memory load/store operation, return true if it has one mask bit set. | |
static SDValue | reduceMaskedLoadToScalarLoad (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert. | |
static SDValue | combineMaskedLoadConstantMask (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineMaskedLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | reduceMaskedStoreToScalarStore (MaskedStoreSDNode *MS, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store. | |
static SDValue | combineMaskedStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineVEXTRACT_STORE (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static bool | isHorizontalBinOp (unsigned HOpcode, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsCommutative, SmallVectorImpl< int > &PostShuffleMask, bool ForceHorizOp) |
Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS. | |
static SDValue | combineToHorizontalAddSub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineFMulcFCMulc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineFaddCFmul (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineFaddFsub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Do target-specific dag combines on floating-point adds/subs. | |
static SDValue | combineLRINT_LLRINT (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineTruncatedArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL) |
Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen. | |
static SDValue | combinePMULH (SDValue Src, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | detectPMADDUBSW (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL) |
static SDValue | combineTruncate (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineVTRUNC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | isFNEG (SelectionDAG &DAG, SDNode *N, unsigned Depth=0) |
Returns the negated value if the node N flips sign of FP value. | |
static unsigned | negateFMAOpcode (unsigned Opcode, bool NegMul, bool NegAcc, bool NegRes) |
static SDValue | combineFneg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
Do target-specific dag combines on floating point negations. | |
static SDValue | lowerX86FPLogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | foldXor1SetCC (SDNode *N, SelectionDAG &DAG) |
Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val) | |
static SDValue | combineXorSubCTLZ (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineXor (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineBITREVERSE (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineAVG (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineBEXTR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static bool | isNullFPScalarOrVectorConst (SDValue V) |
static SDValue | getNullFPConstForNullVal (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value. | |
static SDValue | combineFAndFNotToFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineFAnd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Do target-specific dag combines on X86ISD::FAND nodes. | |
static SDValue | combineFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Do target-specific dag combines on X86ISD::FANDN nodes. | |
static SDValue | combineFOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes. | |
static SDValue | combineFMinFMax (SDNode *N, SelectionDAG &DAG) |
Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes. | |
static SDValue | combineFMinNumFMaxNum (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineX86INT_TO_FP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineCVTP2I_CVTTP2I (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineAndnp (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
Do target-specific dag combines on X86ISD::ANDNP nodes. | |
static SDValue | combineBT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineCVTPH2PS (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineSextInRegCmov (SDNode *N, SelectionDAG &DAG) |
static SDValue | combineSignExtendInReg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | promoteExtBeforeAdd (SDNode *Ext, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
sext(add_nsw(x, C)) --> add(sext(x), C_sext) zext(add_nuw(x, C)) --> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode. | |
static SDValue | combineToExtendCMOV (SDNode *Extend, SelectionDAG &DAG) |
static SDValue | combineExtSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineSext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | getInvertedVectorForFMA (SDValue V, SelectionDAG &DAG) |
static SDValue | combineFMA (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineFMADDSUB (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineZext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | truncateAVX512SetCCNoBWI (EVT VT, EVT OpVT, SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just pre-promote its result type since vXi1 vectors don't get promoted during type legalization. | |
static SDValue | combineSetCC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineMOVMSK (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineTESTP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineX86GatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | rebuildGatherScatter (MaskedGatherScatterSDNode *GorS, SDValue Index, SDValue Base, SDValue Scale, SelectionDAG &DAG) |
static SDValue | combineGatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineX86SetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineBrCond (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
Optimize branch condition evaluation. | |
static SDValue | combineVectorCompareAndMaskUnaryOp (SDNode *N, SelectionDAG &DAG) |
static SDValue | combineToFPTruncExtElt (SDNode *N, SelectionDAG &DAG) |
If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector element with a bitcast. | |
static SDValue | combineUIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineSIntToFP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static bool | needCarryOrOverflowFlag (SDValue Flags) |
static bool | onlyZeroFlagUsed (SDValue Flags) |
static SDValue | combineCMP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineX86AddSub (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &ST) |
static SDValue | combineSBB (SDNode *N, SelectionDAG &DAG) |
static SDValue | combineADC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | matchPMADDWD (SelectionDAG &DAG, SDValue Op0, SDValue Op1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget) |
static SDValue | matchPMADDWD_2 (SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget) |
static SDValue | combineAddOfPMADDWD (SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, EVT VT) |
static SDValue | pushAddIntoCmovOfConsts (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
CMOV of constants requires materializing constant operands in registers. | |
static SDValue | combineAdd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineSubABS (SDNode *N, SelectionDAG &DAG) |
static SDValue | combineSubSetcc (SDNode *N, SelectionDAG &DAG) |
static SDValue | combineX86CloadCstore (SDNode *N, SelectionDAG &DAG) |
static SDValue | combineSub (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineVectorCompare (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static std::optional< unsigned > | CastIntSETCCtoFP (MVT VT, ISD::CondCode CC, unsigned NumSignificantBitsLHS, unsigned NumSignificantBitsRHS) |
static SDValue | combineConcatVectorOps (const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g. | |
static SDValue | combineCONCAT_VECTORS (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineINSERT_SUBVECTOR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | narrowExtractedVectorSelect (SDNode *Ext, const SDLoc &DL, SelectionDAG &DAG) |
If we are extracting a subvector of a vector select and the select condition is composed of concatenated vectors, try to narrow the select width. | |
static SDValue | combineEXTRACT_SUBVECTOR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineScalarToVector (SDNode *N, SelectionDAG &DAG) |
static SDValue | combinePMULDQ (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineVPMADD (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineEXTEND_VECTOR_INREG (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineKSHIFT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineFP16_TO_FP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineFP_EXTEND (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget) |
static SDValue | combineBROADCAST_LOAD (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineFP_ROUND (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget) |
static SDValue | combineMOVDQ2Q (SDNode *N, SelectionDAG &DAG) |
static SDValue | combinePDEP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | FixupMMXIntrinsicTypes (SDNode *N, SelectionDAG &DAG) |
static SDValue | combineINTRINSIC_WO_CHAIN (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineINTRINSIC_W_CHAIN (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static SDValue | combineINTRINSIC_VOID (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI) |
static bool | matchAsm (StringRef S, ArrayRef< const char * > Pieces) |
static bool | clobbersFlagRegisters (const SmallVector< StringRef, 4 > &AsmPieces) |
static X86::CondCode | parseConstraintCode (llvm::StringRef Constraint) |
static bool | isGRClass (const TargetRegisterClass &RC) |
Check if RC is a general purpose register class. | |
static bool | isFRClass (const TargetRegisterClass &RC) |
Check if RC is a vector register class. | |
static bool | isVKClass (const TargetRegisterClass &RC) |
Check if RC is a mask register class. | |
static bool | useEGPRInlineAsm (const X86Subtarget &Subtarget) |
Variables | |
static cl::opt< int > | ExperimentalPrefInnermostLoopAlignment ("x86-experimental-pref-innermost-loop-alignment", cl::init(4), cl::desc("Sets the preferable loop alignment for experiments (as log2 bytes) " "for innermost loops only. If specified, this option overrides " "alignment set by x86-experimental-pref-loop-alignment."), cl::Hidden) |
static cl::opt< int > | BrMergingBaseCostThresh ("x86-br-merging-base-cost", cl::init(2), cl::desc("Sets the cost threshold for when multiple conditionals will be merged " "into one branch versus be split in multiple branches. Merging " "conditionals saves branches at the cost of additional instructions. " "This value sets the instruction cost limit, below which conditionals " "will be merged, and above which conditionals will be split. Set to -1 " "to never merge branches."), cl::Hidden) |
static cl::opt< int > | BrMergingCcmpBias ("x86-br-merging-ccmp-bias", cl::init(6), cl::desc("Increases 'x86-br-merging-base-cost' in cases that the target " "supports conditional compare instructions."), cl::Hidden) |
static cl::opt< int > | BrMergingLikelyBias ("x86-br-merging-likely-bias", cl::init(0), cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "likely, then it is likely that if the conditionals are split " "both sides will be executed, so it may be desirable to increase " "the instruction cost threshold. Set to -1 to never merge likely " "branches."), cl::Hidden) |
static cl::opt< int > | BrMergingUnlikelyBias ("x86-br-merging-unlikely-bias", cl::init(-1), cl::desc("Decreases 'x86-br-merging-base-cost' in cases that it is unlikely " "that all conditionals will be executed. For example for merging " "the conditionals (a == b && c > d), if its known that a == b is " "unlikely, then it is unlikely that if the conditionals are split " "both sides will be executed, so it may be desirable to decrease " "the instruction cost threshold. Set to -1 to never merge unlikely " "branches."), cl::Hidden) |
static cl::opt< bool > | MulConstantOptimization ("mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc."), cl::Hidden) |
const unsigned | X87StateSize = 28 |
const unsigned | FPStateSize = 32 |
const unsigned | FPStateSizeInBits = FPStateSize * 8 |
#define DEBUG_TYPE "x86-isel" |
Definition at line 71 of file X86ISelLowering.cpp.
#define GET_EGPR_IF_ENABLED | ( | OPC | ) | (Subtarget.hasEGPR() ? OPC##_EVEX : OPC) |
enum BitTestKind : unsigned |
Enumerator | |
---|---|
UndefBit | |
ConstantBit | |
NotConstantBit | |
ShiftBit | |
NotShiftBit |
Definition at line 30521 of file X86ISelLowering.cpp.
|
strong |
Different mul shrinking modes.
Enumerator | |
---|---|
MULS8 | |
MULU8 | |
MULS16 | |
MULU16 |
Definition at line 47527 of file X86ISelLowering.cpp.
|
static |
Definition at line 43443 of file X86ISelLowering.cpp.
References adjustBitcastSrcVectorSSE1(), llvm::ISD::AND, llvm::ISD::BITCAST, DL, getAltBitOpcode(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isNormalLoad(), llvm::ISD::OR, llvm::ISD::SETCC, llvm::ISD::SETLT, and llvm::ISD::XOR.
Referenced by adjustBitcastSrcVectorSSE1(), and combineBitcastvxi1().
|
static |
Definition at line 7546 of file X86ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), getUnderlyingExtractedFromVec(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), Idx, llvm::ISD::INSERT_VECTOR_ELT, llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorBase< Size_T >::size(), and llvm::ISD::UNDEF.
|
static |
Definition at line 17551 of file X86ISelLowering.cpp.
References llvm::ISD::ABS, llvm::ISD::ADD, llvm::ISD::AND, llvm::X86Subtarget::hasAVX512(), llvm::ISD::MUL, llvm::ISD::OR, llvm::ISD::SHL, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, llvm::ISD::UMAX, llvm::ISD::UMIN, and llvm::ISD::XOR.
Referenced by commuteSelect(), and lowerVECTOR_SHUFFLE().
|
static |
Check whether a compaction lowering can be done by dropping even/odd elements and compute how many times even/odd elements must be dropped.
This handles shuffles which take every Nth element where N is a power of two. Example shuffle masks:
(even) N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24
(odd) N = 1: 1, 3, 5, 7, 9, 11, 13, 15, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31
Any of these lanes can of course be undef.
This routine only supports N <= 3. FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here for larger N.
Definition at line 10312 of file X86ISelLowering.cpp.
References assert(), llvm::isPowerOf2_32(), N, and llvm::Offset.
Referenced by lowerV16I8Shuffle(), and lowerV8I16Shuffle().
|
static |
Definition at line 50154 of file X86ISelLowering.cpp.
References A, llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), B, llvm::CallingConv::C, DL, getAVX512Node(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::MVT::isVector(), N, llvm::ISD::OR, llvm::peekThroughBitcasts(), llvm::SmallVectorBase< Size_T >::size(), useVPTERNLOG(), llvm::X86ISD::VPTERNLOG, X, and Y.
Referenced by combineOr().
|
static |
Attempt to fold vpermf128(op(),op()) -> op(vpermf128(),vpermf128()).
Definition at line 40668 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::isUndef(), LHS, llvm::X86ISD::MOVDDUP, llvm::peekThroughBitcasts(), llvm::X86ISD::PSHUFD, RHS, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.
Referenced by combineTargetShuffle().
Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.
Definition at line 17489 of file X86ISelLowering.cpp.
References assert(), and Size.
Referenced by combineX86ShuffleChainWithExtract(), combineX86ShufflesRecursively(), and lowerVECTOR_SHUFFLE().
|
static |
Definition at line 39397 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::any_of(), assert(), llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::ArrayRef< T >::empty(), extract128BitVector(), llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::SelectionDAG::getBitcast(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86ISD::HADD, llvm::Hi, llvm::X86ISD::HSUB, isInRange(), llvm::SDNode::isOnlyUserOf(), isRepeatedTargetShuffleMask(), isUndefInRange(), isUndefOrZero(), isUndefOrZeroOrInRange(), LHS, llvm::Lo, llvm::M0(), llvm::M1(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, scaleShuffleElements(), shouldUseHorizontalOp(), llvm::X86ISD::SHUFP, llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, std::swap(), and widenSubVector().
Referenced by combineX86ShufflesRecursively(), and lowerVECTOR_SHUFFLE().
|
static |
Definition at line 40510 of file X86ISelLowering.cpp.
References llvm::X86ISD::BLENDI, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), getTargetConstantFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_SUBVECTOR, llvm::X86ISD::INSERTPS, llvm::TargetLoweringBase::isBinOp(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), isLogicOp(), isShuffleFoldableLoad(), llvm::SelectionDAG::isSplatValue(), isTargetShuffle(), isUnaryOp(), LHS, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSS, N, llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, RHS, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by combineShuffle().
|
static |
Definition at line 47529 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::EVT::getScalarSizeInBits(), N, and llvm::SelectionDAG::SignBitIsZero().
Referenced by matchPMADDWD(), and reduceVMULWidth().
Definition at line 3773 of file X86ISelLowering.cpp.
References scaleShuffleElements().
Referenced by combineBlendOfPermutes(), and combineSetCCMOVMSK().
Definition at line 3738 of file X86ISelLowering.cpp.
References canWidenShuffleElements().
|
static |
Definition at line 3722 of file X86ISelLowering.cpp.
References assert(), canWidenShuffleElements(), llvm::APInt::isZero(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
|
static |
Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.
Appends the mask for wider elements in WidenedMask if valid. Otherwise leaves it in an unspecified state.
NOTE: This must handle normal vector shuffle masks and target vector shuffle masks. The latter have the special property of a '-2' representing a zero-ed lane of a vector.
Definition at line 3672 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::M0(), llvm::M1(), llvm::SmallVectorBase< Size_T >::size(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by canWidenShuffleElements(), combineBlendOfPermutes(), combineExtractWithShuffle(), combineX86ShuffleChainWithExtract(), combineX86ShufflesRecursively(), LowerShift(), lowerShuffleAsBlendAndPermute(), lowerV2X128Shuffle(), lowerV4X128Shuffle(), lowerVECTOR_SHUFFLE(), and scaleShuffleElements().
|
static |
Definition at line 56105 of file X86ISelLowering.cpp.
References assert(), CC, llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::MVT::getScalarType(), llvm::APFloatBase::semanticsPrecision(), llvm::ISD::SETEQ, llvm::ISD::SETGT, and llvm::ISD::SINT_TO_FP.
Referenced by combineConcatVectorOps().
|
static |
Definition at line 23040 of file X86ISelLowering.cpp.
References llvm::ISD::SETONE, and llvm::ISD::SETUEQ.
Referenced by convertIntLogicToFPLogic(), and LowerVSETCC().
|
static |
Definition at line 34856 of file X86ISelLowering.cpp.
References isEFLAGSLiveAfter(), and TRI.
Definition at line 43401 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, checkBitcastSrcVectorSize(), llvm::ISD::FREEZE, llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::OR, llvm::ISD::SELECT, llvm::ISD::SETCC, Size, llvm::ISD::TRUNCATE, llvm::ISD::VSELECT, and llvm::ISD::XOR.
Referenced by checkBitcastSrcVectorSize(), and combineBitcastvxi1().
|
static |
Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
Match: (X86or (X86setcc) (X86setcc)) (X86cmp (and (X86setcc) (X86setcc)), 0)
Definition at line 46755 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::AND, llvm::X86ISD::CMP, Cond, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::isNullConstant(), llvm::ISD::OR, llvm::X86ISD::OR, and llvm::X86ISD::SETCC.
Referenced by combineCMov().
|
static |
Definition at line 46637 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::CallingConv::C, CC, llvm::X86ISD::CMOV, llvm::X86ISD::CMP, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::ConstantSDNode::getZExtValue(), llvm::isOneConstant(), llvm::X86ISD::RDRAND, llvm::X86ISD::RDSEED, llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSetCCEFLAGS().
|
static |
Definition at line 46568 of file X86ISelLowering.cpp.
References llvm::ISD::AND, CC, llvm::X86ISD::CMP, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NS, llvm::X86::COND_S, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getValidShiftAmount(), llvm::isNullConstant(), llvm::APInt::lshrInPlace(), llvm::X86ISD::OR, llvm::ISD::SHL, llvm::SelectionDAG::SignBitIsZero(), and llvm::ISD::SRA.
Referenced by combineSetCCEFLAGS().
|
static |
Definition at line 58282 of file X86ISelLowering.cpp.
References llvm::is_contained(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by llvm::X86TargetLowering::ExpandInlineAsm().
|
static |
Definition at line 4051 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::append(), assert(), collectConcatOps(), llvm::ISD::CONCAT_VECTORS, llvm::SmallVectorBase< Size_T >::empty(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::Hi, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::isNullConstant(), llvm::Lo, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by collectConcatOps(), combineBitcastvxi1(), combineINSERT_SUBVECTOR(), combineSetCCMOVMSK(), combineTargetShuffle(), isFreeToSplitVector(), IsNOT(), and isUpperSubvectorUndef().
|
static |
Definition at line 55412 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::ISD::ADD, llvm::ISD::AND, combineCarryThroughADD(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::X86::COND_B, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), LHS, N, RHS, and llvm::X86ISD::SETCC_CARRY.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 55827 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::ISD::ADD, assert(), combineAddOfPMADDWD(), combineAddOrSubToADCOrSBB(), combineToHorizontalAddSub(), DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDNode::getVTList(), getZeroVector(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDNode::hasOneUse(), llvm::ISD::isBuildVectorAllZeros(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86::isZeroNode(), matchPMADDWD(), matchPMADDWD_2(), N, llvm::X86ISD::PSADBW, pushAddIntoCmovOfConsts(), Select, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::SelectionDAG::willNotOverflowAdd(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 55718 of file X86ISelLowering.cpp.
References DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::APInt::getSplat(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), LHS, llvm::SelectionDAG::MaskedValueIsZero(), RHS, and llvm::X86ISD::VPMADDWD.
Referenced by combineAdd().
|
static |
If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.
This replaces TEST+SET+{ADD/SUB} with CMP+{ADC, SBB}. Also try (ADD/SUB)+(AND(SRL,1)) bit extraction pattern with BT+{ADC, SBB}.
Definition at line 50466 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::ISD::AND, CC, llvm::X86ISD::CMP, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getVTList(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::EVT::isInteger(), llvm::isOneConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86::isZeroNode(), LowerAndToBT(), llvm::X86ISD::SBB, llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::ISD::SETNE, llvm::X86ISD::SUB, X, Y, and llvm::ISD::ZERO_EXTEND.
Referenced by combineAdd(), combineAddOrSubToADCOrSBB(), combineOrXorWithSETCC(), combineSub(), and combineX86AddSub().
|
static |
If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB.
This replaces TEST+SET+{ADD/SUB} with CMP+{ADC, SBB}.
Definition at line 50650 of file X86ISelLowering.cpp.
References combineAddOrSubToADCOrSBB(), DL, llvm::SelectionDAG::getNegative(), N, llvm::ISD::SUB, X, and Y.
|
static |
Definition at line 49888 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::all_of(), llvm::ISD::AND, assert(), llvm::APInt::clearAllBits(), combineAndLoadToBZHI(), combineAndMaskToShift(), combineAndNotIntoANDNP(), combineAndOrForCcmpCtest(), combineAndShuffleNot(), combineBitcastvxi1(), combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineBMILogicOp(), combineCompareEqual(), combineScalarAndWithMaskSetcc(), combineX86ShufflesRecursively(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_AE, llvm::X86::COND_B, convertIntLogicToFPLogic(), llvm::ISD::DELETED_NODE, llvm::APInt::exactLogBase2(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FAND, llvm::APInt::getAllOnes(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), getBitWidth(), getBT(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNegative(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetConstantFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), I, Idx, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isBitwiseNot(), llvm::isConstOrConstSplat(), llvm::isOneConstant(), llvm::SDNode::isOnlyUserOf(), llvm::APInt::isPowerOf2(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), isZero(), LHS, llvm::SelectionDAG::MaskedValueIsZero(), matchScalarReduction(), llvm::X86::MaxShuffleCombineDepth, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, N, RHS, llvm::APInt::setBit(), llvm::ISD::SETEQ, llvm::ISD::SHL, llvm::TargetLowering::SimplifyDemandedBits(), llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::APInt::uge(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 49520 of file X86ISelLowering.cpp.
References llvm::AllOnes, llvm::ISD::AND, GEP, llvm::SelectionDAG::getAllOnesConstant(), llvm::Type::getArrayElementType(), llvm::Type::getArrayNumElements(), llvm::SelectionDAG::getConstant(), getIndexFromUnindexedLoad(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::Type::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MachineMemOperand::getValue(), llvm::SelectionDAG::getZExtOrTrunc(), hasBZHI(), llvm::Type::isIntegerTy(), N, llvm::ISD::SRL, llvm::ISD::SUB, and llvm::ISD::TRUNCATE.
Referenced by combineAnd().
|
static |
If this is a zero/all-bits result that is bitwise-anded with a low bits mask.
(Mask == 1 for the x86 lowering of a SETCC + ZEXT), replace the 'and' with a shift-right to eliminate loading the vector constant mask value.
Definition at line 49425 of file X86ISelLowering.cpp.
References llvm::X86ISD::ANDNP, llvm::SelectionDAG::ComputeNumSignBits(), llvm::APInt::countr_one(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::isAllOnesOrAllOnesSplat(), llvm::isBitwiseNot(), llvm::X86::isConstantSplat(), llvm::EVT::isInteger(), llvm::APInt::isMask(), llvm::EVT::isSimple(), N, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), llvm::ISD::SRA, llvm::ISD::SRL, supportedVectorShiftWithImm(), llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, X, and Y.
Referenced by combineAnd().
|
static |
Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
Definition at line 49025 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), IsNOT(), N, X, and Y.
Referenced by combineAnd().
|
static |
Do target-specific dag combines on X86ISD::ANDNP nodes.
Definition at line 53498 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::AND, llvm::ISD::BITCAST, llvm::APInt::clearAllBits(), combineX86ShufflesRecursively(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::DELETED_NODE, DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), I, llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), IsNOT(), llvm::SDValue::isUndef(), llvm::MVT::isVector(), isZero(), N, llvm::ISD::OR, llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::APInt::setBit(), llvm::TargetLowering::SimplifyDemandedBits(), and llvm::TargetLowering::SimplifyDemandedVectorElts().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 49808 of file X86ISelLowering.cpp.
References llvm::X86ISD::CCMP, llvm::X86ISD::CMP, llvm::X86::COND_NP, llvm::X86::COND_P, llvm::X86ISD::CTEST, DL, llvm::SDNode::getAsAPIntVal(), llvm::X86::getCCMPCondFlagsFromCondCode(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::APInt::getSExtValue(), llvm::SelectionDAG::getTargetConstant(), llvm::isNullConstant(), N, llvm::ISD::OR, llvm::X86ISD::SETCC, llvm::X86ISD::SUB, and std::swap().
Referenced by combineAnd(), and combineOr().
|
static |
Try to fold: and (vector_shuffle<Z,...,Z> (insert_vector_elt undef, (xor X, -1), Z), undef), Y -> andnp (vector_shuffle<Z,...,Z> (insert_vector_elt undef, X, Z), undef), Y.
Definition at line 49056 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), IsNOT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), N, llvm::peekThroughOneUseBitcasts(), splitVector(), llvm::X86Subtarget::useAVX512Regs(), X, and Y.
Referenced by combineAnd().
|
static |
Try to convert a vector reduction sequence composed of binops and shuffles into horizontal ops.
Definition at line 45026 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::computeKnownBits(), llvm::ISD::CONCAT_VECTORS, DL, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::KnownBits::getMaxValue(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::ISD::INSERT_VECTOR_ELT, llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::Lo, llvm::Log2_32(), llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::MUL, llvm::X86ISD::PACKUS, llvm::X86ISD::PSADBW, shouldUseHorizontalOp(), SplitOpsAndApply(), splitVector(), llvm::ISD::TRUNCATE, llvm::APInt::ule(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineExtractVectorElt().
|
static |
Definition at line 53196 of file X86ISelLowering.cpp.
References llvm::ISD::AVGCEILS, llvm::ISD::AVGCEILU, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::APInt::getSignMask(), llvm::EVT::isVector(), N, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 44601 of file X86ISelLowering.cpp.
References llvm::ISD::ABS, llvm::ISD::ADD, llvm::ISD::ANY_EXTEND, createPSADBW(), detectZextAbsDiff(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::isPowerOf2_32(), llvm::Log2_32(), llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by combineExtractVectorElt().
|
static |
Definition at line 53220 of file X86ISelLowering.cpp.
References llvm::APInt::getAllOnes(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), N, and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 43899 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, llvm::EVT::changeVectorElementTypeToInteger(), combineBitcastToBoolVector(), combineBitcastvxi1(), combineCastedMaskArithmetic(), combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::countl_zero(), createMMXBuildVector(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, F64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_TO_SINT, llvm::X86ISD::FXOR, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::APFloatBase::IEEEdouble(), llvm::EVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::isNullConstantOrUndef(), llvm::isPowerOf2_32(), llvm::EVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86ISD::MMX_MOVW2D, llvm::X86ISD::MOVDQ2Q, llvm::X86ISD::MOVMSK, N, llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::ISD::OR, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SETLT, llvm::ISD::TRUNCATE, llvm::X86ISD::VBROADCAST_LOAD, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 43810 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::CallingConv::C, combineBitcastToBoolVector(), llvm::ISD::Constant, llvm::Depth, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86ISD::KSHIFTL, llvm::SelectionDAG::MaxRecursionDepth, llvm::ISD::OR, llvm::ISD::SHL, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by combineBitcast(), and combineBitcastToBoolVector().
|
static |
Definition at line 43508 of file X86ISelLowering.cpp.
References adjustBitcastSrcVectorSSE1(), llvm::all_of(), llvm::ISD::ANY_EXTEND, llvm::SmallVectorTemplateCommon< T, typename >::begin(), checkBitcastSrcVectorSize(), collectConcatOps(), combineBitcastvxi1(), DL, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), getPMOVMSKB(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isSimple(), llvm::X86ISD::MOVMSK, llvm::ISD::SETCC, llvm::ISD::SETLT, llvm::ISD::SIGN_EXTEND, signExtendBitcastSrcVector(), llvm::MVT::SimpleTy, llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::TRUNCATE, and widenSubVector().
Referenced by combineAnd(), combineBitcast(), combineBitcastvxi1(), combineExtractVectorElt(), combineOr(), and combinePredicateReduction().
|
static |
Definition at line 49304 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), convertIntLogicToFPLogicOpcode(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::isFloatingPoint(), llvm::X86ISD::MOVMSK, N, llvm::ISD::OR, and llvm::ISD::XOR.
Referenced by combineAnd(), combineOr(), and combineXor().
|
static |
Definition at line 49382 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::hasOneUse(), LHS, N, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::peekThroughOneUseBitcasts(), RHS, and llvm::ISD::XOR.
Referenced by combineAnd(), combineOr(), and combineXor().
|
static |
Definition at line 49339 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), N, llvm::ISD::OR, llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and llvm::ISD::XOR.
Referenced by combineAnd(), combineOr(), and combineXor().
|
static |
Definition at line 53168 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSSE3(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), and N.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 40405 of file X86ISelLowering.cpp.
References assert(), canScaleShuffleElements(), canWidenShuffleElements(), DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::getShuffleDemandedElts(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), I, llvm::APInt::intersects(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), isBlendOrUndef(), isLaneCrossingShuffleMask(), isUndefOrEqual(), isUndefOrInRange(), llvm::APInt::isZero(), llvm::peekThroughOneUseBitcasts(), scaleShuffleElements(), and llvm::SM_SentinelUndef.
Referenced by llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
|
static |
Definition at line 49727 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), getBMIMatchingOp(), llvm::EVT::isScalarInteger(), N, and llvm::ISD::XOR.
Referenced by combineAnd(), and combineXor().
|
static |
Optimize branch condition evaluation.
Definition at line 54844 of file X86ISelLowering.cpp.
References llvm::X86ISD::BRCOND, CC, combineSetCCEFLAGS(), Cond, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), and N.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 57640 of file X86ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::EVT::getFixedSizeInBits(), llvm::EVT::getSizeInBits(), N, Ptr, llvm::X86ISD::SUBV_BROADCAST_LOAD, and llvm::X86ISD::VBROADCAST_LOAD.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 53631 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::BitWidth, llvm::ISD::DELETED_NODE, llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::Log2_32(), N, and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 46796 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADD, llvm::ISD::AND, llvm::X86::COND_A, llvm::X86::COND_B, llvm::X86::COND_E, DL, getBT(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::SDValue::getValueType(), llvm::SDNode::getVTList(), llvm::SDNode::hasOneUse(), llvm::isAllOnesConstant(), llvm::EVT::isInteger(), llvm::isOneConstant(), llvm::X86ISD::SETCC, llvm::X86ISD::SETCC_CARRY, llvm::ISD::SRL, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 43674 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, combinevXi1ConstantToInteger(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX512(), llvm::Value::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isScalarInteger(), llvm::EVT::isVector(), LHS, N, llvm::ISD::OR, RHS, and llvm::ISD::XOR.
Referenced by combineBitcast().
|
static |
Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL].
Definition at line 47277 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::Add, llvm::ISD::ADD, assert(), llvm::X86Subtarget::canUseCMOV(), CC, checkBoolTestAndOrSetCCCombine(), llvm::X86ISD::CMOV, llvm::X86ISD::CMP, combineSetCCEFLAGS(), Cond, llvm::X86::COND_AE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::ISD::CTTZ, llvm::ISD::CTTZ_ZERO_UNDEF, DL, llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::APInt::getZExtValue(), hasFPCMov(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::ISD::MUL, N, llvm::ISD::SHL, llvm::ISD::SUB, llvm::X86ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, llvm::APInt::ult(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 55200 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::ISD::AND, llvm::X86ISD::AND, llvm::BitWidth, llvm::X86ISD::CMP, combineX86SubCmpForFlags(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::APInt::getZExtValue(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86ISD::KSHIFTR, llvm::SelectionDAG::MaskedValueIsZero(), N, needCarryOrOverflowFlag(), onlyZeroFlagUsed(), llvm::ISD::OR, llvm::X86ISD::OR, llvm::ISD::SHL, llvm::ISD::SRL, llvm::ISD::SUB, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, llvm::APInt::ult(), widenMaskVector(), llvm::ISD::XOR, llvm::X86ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 40350 of file X86ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::X86::mayFoldLoad(), N, llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::SHUFP, and llvm::X86ISD::VPERMILPI.
Referenced by combineTargetShuffle().
|
static |
Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends.
Likewise for OR -> CMPNEQSS.
Definition at line 48918 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BR_CC, llvm::ISD::BRCOND, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_P, llvm::ISD::CopyToReg, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FCMP, llvm::X86ISD::FSETCC, llvm::X86ISD::FSETCCM, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, isAndOrOfSetCCs(), N, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SELECT, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineAnd(), and combineOr().
|
static |
Definition at line 56793 of file X86ISelLowering.cpp.
References llvm::CallingConv::C, combineConcatVectorOps(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX(), I, llvm::TargetLoweringBase::isTypeLegal(), N, llvm::peekThroughBitcasts(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.
ISD::INSERT_SUBVECTOR). The ops are assumed to be of the same type.
Definition at line 56127 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::all_equal(), llvm::all_of(), llvm::X86TargetLowering::allowsMemoryAccess(), llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::SmallVectorImpl< T >::append(), assert(), llvm::X86ISD::BLENDI, llvm::X86ISD::BLENDV, llvm::CallingConv::C, CastIntSETCCtoFP(), llvm::EVT::changeVectorElementType(), llvm::X86ISD::CMPP, llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::ISD::CONCAT_VECTORS, concatSubVectors(), llvm::ISD::CTLZ, llvm::ISD::CTLZ_ZERO_UNDEF, llvm::ISD::CTPOP, llvm::ISD::CTTZ, llvm::ISD::CTTZ_ZERO_UNDEF, DL, EltsFromConsecutiveLoads(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::ISD::FADD, llvm::CallingConv::Fast, llvm::ISD::FDIV, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), getBROADCAST_LOAD(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), getConstantVector(), getConstVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getEntryNode(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::X86Subtarget::getPreferVectorWidth(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), getTargetConstantFromNode(), llvm::X86Subtarget::getTargetLowering(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), getZeroVector(), llvm::X86ISD::GF2P8AFFINEQB, llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86ISD::HSUB, I, Idx, llvm::APInt::insertBits(), llvm::EVT::is256BitVector(), llvm::MVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isFloatingPoint(), llvm::MVT::isFloatingPoint(), llvm::ISD::isNormalLoad(), llvm::EVT::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LHS, llvm::X86::mayFoldLoad(), llvm::X86::mayFoldLoadIntoBroadcastFromMem(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::ISD::MUL, llvm::none_of(), llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), llvm::X86ISD::PSADBW, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SETEQ, llvm::ISD::SETGT, llvm::ISD::SETOEQ, llvm::ISD::SETOGT, llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::SUB, llvm::X86ISD::SUBV_BROADCAST_LOAD, translateX86FSETCC(), llvm::ISD::TRUNCATE, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86Subtarget::useAVX512Regs(), llvm::X86Subtarget::useBWIRegs(), llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPMADDUBSW, llvm::X86ISD::VPMADDWD, llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::ISD::VSELECT, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, llvm::X86ISD::VSRLI, widenSubVector(), and llvm::ISD::XOR.
Referenced by combineCONCAT_VECTORS(), and combineINSERT_SUBVECTOR().
|
static |
Definition at line 51174 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::EVT::getFixedSizeInBits(), llvm::Type::getPrimitiveSizeInBits(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetConstantFromBasePtr(), llvm::Value::getType(), llvm::SDNode::getValueType(), llvm::X86Subtarget::hasAVX(), I, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::ISD::isNormalLoad(), N, llvm::ISD::NON_EXTLOAD, Ptr, llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::SDNode::uses(), and llvm::X86ISD::VBROADCAST_LOAD.
Referenced by combineLoad().
|
static |
Definition at line 53461 of file X86ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getFloatingPointVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getValue(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::ISD::isNormalLoad(), N, narrowLoadToVZLoad(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), and llvm::SelectionDAG::ReplaceAllUsesOfValueWith().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 53647 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::DELETED_NODE, llvm::SelectionDAG::getBitcast(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::ISD::isNormalLoad(), N, narrowLoadToVZLoad(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::TargetLowering::SimplifyDemandedVectorElts(), and llvm::X86ISD::STRICT_CVTPH2PS.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 57435 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::EVT::changeVectorElementType(), combineX86ShufflesRecursively(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getExtLoad(), llvm::SDNode::getFlags(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getOpcode_EXTEND(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE41(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isLoadExtLegal(), llvm::ISD::isNormalLoad(), llvm::TargetLoweringBase::isTypeLegal(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::ZERO_EXTEND_VECTOR_INREG, and llvm::ISD::ZEXTLOAD.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 57040 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, llvm::X86ISD::CMPP, Concat, llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTUI2P, DL, llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::ISD::FP_EXTEND, llvm::ISD::FP_TO_SINT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isBitwiseNot(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isExtOpcode(), llvm::ISD::isExtVecInRegOpcode(), llvm::SelectionDAG::isSplatValue(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86ISD::MOVDDUP, N, narrowExtractedVectorSelect(), llvm::SDNode::ops(), llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), scaleShuffleElements(), llvm::ISD::SINT_TO_FP, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, splitVectorIntBinary(), llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::ISD::TRUNCATE, llvm::ISD::UINT_TO_FP, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VFPEXT, llvm::ISD::VSELECT, llvm::X86ISD::VSHLI, and llvm::X86ISD::VSRLI.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 44684 of file X86ISelLowering.cpp.
References llvm::any_of(), assert(), llvm::commonAlignment(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getLoad(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::TargetLowering::getVectorElementPointer(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MachinePointerInfo::getWithOffset(), Idx, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), and N.
Referenced by combineExtractVectorElt(), and combineExtractWithShuffle().
|
static |
Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.
For i686, the best sequence is apparently storing the value and loading scalars back, while for x64 we should use 64-bit extracts and shifts.
Definition at line 45219 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::ISD::AND, assert(), llvm::ISD::BITCAST, combineArithReduction(), combineBasicSADPattern(), combineBitcastvxi1(), combineExtractFromVectorLoad(), combineExtractWithShuffle(), combineMinMaxReduction(), combinePredicateReduction(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineVPDPBUSDPattern(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), Idx, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::TargetLoweringBase::isTypeLegal(), llvm::X86ISD::MMX_MOVD2W, N, llvm::peekThroughBitcasts(), llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), scalarizeExtEltFP(), llvm::ISD::SETEQ, llvm::ISD::SHL, llvm::TargetLowering::SimplifyDemandedBits(), llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::TRUNCATE, and llvm::SDNode::uses().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 44721 of file X86ISelLowering.cpp.
References llvm::any_of(), assert(), canWidenShuffleElements(), combineExtractFromVectorLoad(), extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::APInt::getZExtValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, Idx, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::EVT::isScalarInteger(), isUndefOrZeroInRange(), llvm::Lo, N, llvm::narrowShuffleMaskElts(), llvm::Offset, llvm::peekThroughBitcasts(), llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::APInt::uge(), llvm::APInt::ult(), llvm::APInt::urem(), llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VBROADCAST_LOAD.
Referenced by combineExtractVectorElt().
|
static |
Definition at line 53918 of file X86ISelLowering.cpp.
References CC, llvm::EVT::changeVectorElementTypeToInteger(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86Subtarget::hasAVX512(), llvm::ISD::isUnsignedIntSetCC(), llvm::EVT::isVector(), N, llvm::ISD::SETCC, Size, llvm::X86Subtarget::useAVX512Regs(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineSext(), and combineZext().
|
static |
Definition at line 52247 of file X86ISelLowering.cpp.
References llvm::TargetOptions::AllowFPOpFusion, llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), llvm::ISD::FADD, llvm::FPOpFusion::Fast, llvm::SelectionDAG::getBitcast(), llvm::SDNode::getFlags(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTarget(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllZeros(), LHS, N, llvm::TargetOptions::NoSignedZerosFPMath, llvm::TargetMachine::Options, RHS, llvm::X86ISD::VFCMADDC, llvm::X86ISD::VFCMULC, llvm::X86ISD::VFMADDC, and llvm::X86ISD::VFMULC.
Referenced by combineFaddFsub().
|
static |
Do target-specific dag combines on floating-point adds/subs.
Definition at line 52322 of file X86ISelLowering.cpp.
References combineFaddCFmul(), combineToHorizontalAddSub(), and N.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on X86ISD::FAND nodes.
Definition at line 53290 of file X86ISelLowering.cpp.
References combineFAndFNotToFAndn(), getNullFPConstForNullVal(), lowerX86FPLogicOp(), and N.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 53258 of file X86ISelLowering.cpp.
References llvm::CallingConv::C, DL, llvm::X86ISD::FANDN, llvm::X86ISD::FXOR, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), and N.
Referenced by combineFAnd().
|
static |
Do target-specific dag combines on X86ISD::FANDN nodes.
Definition at line 53307 of file X86ISelLowering.cpp.
References getNullFPConstForNullVal(), isNullFPScalarOrVectorConst(), lowerX86FPLogicOp(), and N.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 54066 of file X86ISelLowering.cpp.
References A, assert(), B, llvm::CallingConv::C, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FMA, llvm::ISD::FMUL, llvm::TargetLowering::getCheaperNegatedExpression(), llvm::MachineFunction::getFunction(), getInvertedVectorForFMA(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasAnyFMA(), llvm::Function::hasOptSize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::isNullConstant(), llvm::TargetLoweringBase::isOperationExpand(), llvm::TargetLoweringBase::isTypeLegal(), N, and negateFMAOpcode().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 54154 of file X86ISelLowering.cpp.
References llvm::TargetLowering::getCheaperNegatedExpression(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasOptSize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, and negateFMAOpcode().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
Definition at line 53341 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::FMAX, llvm::X86ISD::FMAXC, llvm::X86ISD::FMIN, llvm::X86ISD::FMINC, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTarget(), llvm_unreachable, N, llvm::TargetOptions::NoNaNsFPMath, llvm::TargetOptions::NoSignedZerosFPMath, and llvm::TargetMachine::Options.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 53362 of file X86ISelLowering.cpp.
References DL, llvm::X86ISD::FMAX, llvm::ISD::FMAXNUM, llvm::X86ISD::FMIN, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasMinSize(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::SelectionDAG::isKnownNeverNaN(), isSoftF16(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, llvm::TargetOptions::NoNaNsFPMath, llvm::TargetMachine::Options, and llvm::ISD::SETUO.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 52206 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::getBitcast(), llvm::KnownBits::getBitWidth(), llvm::KnownBits::getConstant(), llvm::SelectionDAG::getNode(), llvm::Value::hasOneUse(), llvm::KnownBits::isConstant(), LHS, N, RHS, std::swap(), llvm::X86ISD::VFCMULC, llvm::X86ISD::VFMULC, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on floating point negations.
Definition at line 52873 of file X86ISelLowering.cpp.
References DL, llvm::ISD::FMUL, llvm::X86ISD::FNMSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantFP(), llvm::SDNode::getFlags(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::TargetLowering::getNegatedExpression(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDNodeFlags::hasNoSignedZeros(), llvm::Function::hasOptSize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isFNEG(), llvm::TargetLoweringBase::isTypeLegal(), and N.
Referenced by combineFOr(), combineXor(), and llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
Definition at line 53321 of file X86ISelLowering.cpp.
References assert(), combineFneg(), llvm::X86ISD::FOR, llvm::X86ISD::FXOR, isNullFPScalarOrVectorConst(), lowerX86FPLogicOp(), and N.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 57521 of file X86ISelLowering.cpp.
References llvm::X86ISD::CVTPH2PS, llvm::X86ISD::CVTPS2PH, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP_TO_FP16, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), N, and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 57543 of file X86ISelLowering.cpp.
References assert(), llvm::EVT::changeTypeToInteger(), llvm::EVT::changeVectorElementType(), llvm::EVT::changeVectorElementTypeToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTPH2PS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), N, llvm::ISD::SHL, llvm::X86ISD::STRICT_CVTPH2PS, llvm::ISD::STRICT_FP_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 57676 of file X86ISelLowering.cpp.
References assert(), llvm::EVT::changeVectorElementTypeToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTPS2PH, llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTUI2P, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::isPowerOf2_32(), llvm::SDNode::isStrictFPOpcode(), llvm::EVT::isVector(), N, llvm::ISD::SINT_TO_FP, llvm::X86ISD::STRICT_CVTPS2PH, llvm::X86ISD::STRICT_CVTSI2P, llvm::X86ISD::STRICT_CVTUI2P, and llvm::ISD::STRICT_SINT_TO_FP.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 54720 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::sampleprof::Base, llvm::CallingConv::C, llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::DELETED_NODE, DL, llvm::SDNode::getAsZExtVal(), llvm::SelectionDAG::getConstant(), llvm::BuildVectorSDNode::getConstantSplatNode(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getSplatBuildVector(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::BuildVectorSDNode::isConstant(), llvm::isOneConstant(), N, llvm::BitVector::none(), rebuildGatherScatter(), llvm::ISD::SIGN_EXTEND, llvm::TargetLowering::SimplifyDemandedBits(), llvm::Splat, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 48350 of file X86ISelLowering.cpp.
References llvm::all_of(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::SmallVectorBase< Size_T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getSplitVectorSrc(), getTargetShuffleInputs(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasInt256(), llvm::Hi, Idx, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), isAnyZero(), llvm::EVT::isFloatingPoint(), isHorizOp(), LHS, llvm::Lo, N, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), resolveTargetShuffleInputsAndMask(), RHS, scaleShuffleElements(), llvm::SelectionDAG::SplitVector(), and std::swap().
Referenced by combineVectorHADDSUB(), and combineVectorPack().
|
static |
Definition at line 56829 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), collectConcatOps(), combineConcatVectorOps(), combineX86ShufflesRecursively(), llvm::ISD::EXTRACT_SUBVECTOR, getBROADCAST_LOAD(), llvm::SDValue::getConstantOperandVal(), llvm::MVT::getFixedSizeInBits(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllZeros(), llvm::isNullConstant(), isTargetShuffle(), llvm::SDValue::isUndef(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VBROADCAST_LOAD.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 57864 of file X86ISelLowering.cpp.
References FixupMMXIntrinsicTypes(), llvm::getIntrinsicWithChain(), llvm::INTR_TYPE_CAST_MMX, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, and llvm::IntrinsicData::Type.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 57850 of file X86ISelLowering.cpp.
References FixupMMXIntrinsicTypes(), llvm::getIntrinsicWithChain(), llvm::INTR_TYPE_CAST_MMX, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, and llvm::IntrinsicData::Type.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 57836 of file X86ISelLowering.cpp.
References FixupMMXIntrinsicTypes(), llvm::getIntrinsicWithoutChain(), llvm::INTR_TYPE_CAST_MMX, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, and llvm::IntrinsicData::Type.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 57503 of file X86ISelLowering.cpp.
References llvm::APInt::getAllOnes(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorAllZeros(), N, and llvm::TargetLowering::SimplifyDemandedVectorElts().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 51249 of file X86ISelLowering.cpp.
References llvm::TargetLoweringBase::allowsMemoryAccess(), combineConstantPoolLoads(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, extractSubVector(), llvm::CallingConv::Fast, llvm::SelectionDAG::getAddrSpaceCast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getExtLoad(), llvm::TypeSize::getFixed(), llvm::EVT::getFixedSizeInBits(), llvm::SDNode::getFlags(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarType(), llvm::SDNode::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, llvm::ISD::NON_EXTLOAD, Ptr, llvm::X86AS::PTR32_SPTR, llvm::X86AS::PTR32_UPTR, llvm::X86AS::PTR64, llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::ISD::TokenFactor, and llvm::SDNode::uses().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 45814 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isInteger(), llvm::TargetLoweringBase::isOperationLegal(), N, llvm::ISD::SUB, std::swap(), X, llvm::ISD::XOR, and Y.
Referenced by combineLogicBlendIntoPBLENDV(), and combineSelect().
|
static |
Definition at line 50258 of file X86ISelLowering.cpp.
References assert(), combineLogicBlendIntoConditionalNegate(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::isInteger(), matchLogicBlend(), N, llvm::ISD::OR, llvm::peekThroughBitcasts(), X, and Y.
Referenced by combineOr().
|
static |
Definition at line 52333 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTP2SI, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), and N.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 51511 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), combineMaskedLoadConstantMask(), llvm::ISD::DELETED_NODE, llvm::SelectionDAG::getMaskedLoad(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasAVX512(), N, llvm::ISD::NON_EXTLOAD, reduceMaskedLoadToScalarLoad(), llvm::TargetLowering::SimplifyDemandedBits(), and llvm::TargetLowering::SimplifyMultipleUseDemandedBits().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 51463 of file X86ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMaskedLoad(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isNullConstant(), and ML.
Referenced by combineMaskedLoad().
|
static |
Definition at line 51590 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::DELETED_NODE, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::MaskedStoreSDNode::getOffset(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MaskedStoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::Value::hasOneUse(), llvm::MaskedStoreSDNode::isCompressingStore(), llvm::MaskedStoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegal(), N, reduceMaskedStoreToScalarStore(), llvm::TargetLowering::SimplifyDemandedBits(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), and llvm::ISD::TRUNCATE.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 44317 of file X86ISelLowering.cpp.
References assert(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, llvm::Lo, llvm::SelectionDAG::matchBinOpReduction(), llvm::X86ISD::PHMINPOS, llvm::ISD::SMAX, llvm::ISD::SMIN, splitVector(), llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::Upper, and llvm::ISD::XOR.
Referenced by combineExtractVectorElt().
|
static |
Definition at line 57767 of file X86ISelLowering.cpp.
References llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::SDValue::getValue(), llvm::ISD::isNormalLoad(), llvm::MemSDNode::isSimple(), N, and llvm::SelectionDAG::ReplaceAllUsesOfValueWith().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 54545 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMaxPopulation(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftByConstNode(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), llvm::X86Subtarget::hasSSE2(), Idx, llvm::ISD::isBitwiseLogicOp(), llvm::ISD::isBuildVectorAllOnes(), IsNOT(), llvm::KnownBits::isZero(), llvm::X86ISD::MOVMSK, N, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughOneUseBitcasts(), llvm::TargetLowering::SimplifyDemandedBits(), llvm::X86ISD::VSHLI, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 47902 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::CallingConv::C, combineMulSpecial(), combineMulToPMADDWD(), combineMulToPMULDQ(), DL, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getTargetConstantFromNode(), llvm::Function::hasMinSize(), INT64_MIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm::isConstOrConstSplat(), llvm::EVT::isInteger(), llvm::isPowerOf2_64(), llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::Log2_64(), llvm::maxUIntN(), llvm::X86ISD::MUL_IMM, MulConstantOptimization, N, reduceVMULWidth(), llvm::ISD::SHL, llvm::ISD::SUB, and std::swap().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 47660 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::countr_zero(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isPowerOf2_64(), llvm::Log2_64(), llvm::X86ISD::MUL_IMM, N, llvm::ISD::SHL, and llvm::ISD::SUB.
Referenced by combineMul().
|
static |
Definition at line 47749 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::ComputeMaxSignificantBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, SplitOpsAndApply(), llvm::X86ISD::VPMADDWD, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by combineMul().
|
static |
Definition at line 47860 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::X86ISD::PMULDQ, llvm::X86ISD::PMULUDQ, and SplitOpsAndApply().
Referenced by combineMul().
|
static |
Definition at line 50719 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::AND, assert(), canonicalizeBitSelect(), combineAndOrForCcmpCtest(), combineBitcastvxi1(), combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineCompareEqual(), combineLogicBlendIntoPBLENDV(), combineOrXorWithSETCC(), combineX86ShufflesRecursively(), llvm::ISD::CONCAT_VECTORS, Cond, convertIntLogicToFPLogic(), llvm::ISD::DELETED_NODE, extractSubVector(), foldMaskedMerge(), llvm::X86ISD::FOR, llvm::SelectionDAG::getBitcast(), getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::EVT::getScalarSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), llvm::EVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86ISD::KSHIFTL, llvm::SelectionDAG::MaskedVectorIsZero(), matchScalarReduction(), llvm::ISD::MUL, N, llvm::ISD::OR, llvm::APInt::setBit(), llvm::X86ISD::SETCC, llvm::ISD::SETNE, llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::SUB, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 50334 of file X86ISelLowering.cpp.
References llvm::X86ISD::CMP, llvm::X86::COND_E, llvm::SmallVectorBase< Size_T >::empty(), llvm::SelectionDAG::getNode(), llvm::X86Subtarget::getTargetLowering(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::X86TargetLowering::isCtlzFast(), llvm::isNullConstant(), LHS, lowerX86CmpEqZeroToCtlzSrl(), N, llvm::ISD::OR, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::X86ISD::SETCC, std::swap(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineZext().
|
static |
Definition at line 50670 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), combineAddOrSubToADCOrSBB(), DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), getTargetConstantBitsFromNode(), llvm::SDValue::hasOneUse(), I, llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), N, llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::SETCC, llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by combineOr(), and combineXor().
|
static |
Definition at line 57788 of file X86ISelLowering.cpp.
References llvm::APInt::getAllOnes(), llvm::SelectionDAG::getTargetLoweringInfo(), N, and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 57337 of file X86ISelLowering.cpp.
References llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVectorShuffle(), llvm::Value::hasOneUse(), llvm::ISD::isBuildVectorAllZeros(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), LHS, N, RHS, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::TargetLowering::SimplifyDemandedBits(), and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 52435 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::KnownBits::countMaxActiveBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::EVT::is256BitVector(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), LHS, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, RHS, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineTruncate().
|
static |
Definition at line 44391 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::BitWidth, CC, combineBitcastvxi1(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MVT::getFloatingPointVT(), llvm::SelectionDAG::getFreeze(), llvm::EVT::getIntegerVT(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), getPMOVMSKB(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::Hi, llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), LHS, llvm::Lo, LowerVectorAllEqual(), llvm::Match, llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::OR, llvm::ISD::PARITY, RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::SelectionDAG::SplitVector(), llvm::ISD::TRUNCATE, and llvm::ISD::XOR.
Referenced by combineExtractVectorElt().
|
static |
If we are inverting an PTEST/TESTP operand, attempt to adjust the CC to avoid the inversion.
Definition at line 46855 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), CC, llvm::X86ISD::CMP, llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_INVALID, llvm::X86::COND_NE, DL, llvm::X86ISD::FAND, llvm::X86ISD::FANDN, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getDoubleNumVectorElementsVT(), llvm::MVT::getFloatingPointVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPMOVMSKB(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getSplitVectorSrc(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::EVT::is128BitVector(), llvm::MVT::is128BitVector(), llvm::ISD::isBuildVectorAllOnes(), IsNOT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::ISD::OR, llvm::peekThroughBitcasts(), llvm::X86ISD::PTEST, llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), and llvm::X86ISD::TESTP.
Referenced by combineSetCCEFLAGS().
|
static |
Search for a combinable shuffle across a chain ending in pshufd.
We walk up the chain and look for a combinable shuffle, skipping over shuffles that we could hoist this shuffle's transformation past without altering anything.
Definition at line 40220 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, DL, llvm::SmallVectorBase< Size_T >::empty(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getPSHUFShuffleMask(), getV4X86ShuffleImm8ForMask(), llvm_unreachable, N, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by combineTargetShuffle().
|
static |
Definition at line 55390 of file X86ISelLowering.cpp.
References combineCarryThroughADD(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::isNullConstant(), LHS, N, RHS, llvm::X86ISD::SBB, and llvm::ISD::SUB.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 49602 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BITCAST, Concat, llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::EVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, and llvm::ISD::SETCC.
Referenced by combineAnd().
|
static |
Definition at line 57254 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::ISD::EXTLOAD, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::User::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::KnownBits::isConstant(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::X86ISD::MOVQ2DQ, N, llvm::peekThroughOneUseBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VZEXT_MOVL, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Do target-specific dag combines on SELECT and VSELECT nodes.
Definition at line 45891 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::BITCAST, llvm::X86ISD::BLENDV, llvm::CallingConv::C, CC, llvm::EVT::changeVectorElementType(), llvm::EVT::changeVectorElementTypeToInteger(), combineLogicBlendIntoConditionalNegate(), combineSelectOfTwoConstants(), combineToExtendBoolVectorInReg(), combineVSelectToBLENDV(), combineVSelectWithAllOnesOrZeros(), combinevXi1ConstantToInteger(), commuteSelect(), llvm::SelectionDAG::ComputeNumSignBits(), Cond, createShuffleMaskFromVSELECT(), DL, llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::X86ISD::FMAX, llvm::X86ISD::FMIN, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::Value::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, insertSubVector(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::isConstOrConstSplat(), llvm::SelectionDAG::isEqualTo(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::SelectionDAG::isKnownNeverZeroFloat(), IsNOT(), llvm::isNullConstant(), llvm::isNullOrNullSplat(), llvm::isOneConstant(), isSoftF16(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), isUndefOrZero(), llvm::EVT::isVector(), LHS, llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), llvm::ISD::matchUnaryPredicate(), N, narrowVectorSelect(), llvm::TargetOptions::NoSignedZerosFPMath, llvm::TargetMachine::Options, llvm::ISD::OR, llvm::X86ISD::PCMPGT, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::ISD::SELECT, Select, llvm::ISD::SETCC, llvm::ISD::SETCC_INVALID, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::simplifySelect(), llvm::ISD::SRL, supportedVectorVarShift(), std::swap(), llvm::ISD::TRUNCATE, llvm::ISD::VSELECT, llvm::X86ISD::VSHLV, and llvm::X86ISD::VSRLV.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 45629 of file X86ISelLowering.cpp.
References llvm::APInt::abs(), llvm::ISD::ADD, CC, Cond, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::isNullConstant(), llvm::APInt::isOne(), llvm::APInt::isPowerOf2(), llvm::TargetLoweringBase::isTypeLegal(), LHS, llvm::ISD::MUL, N, RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, std::swap(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineSelect().
|
static |
Definition at line 54256 of file X86ISelLowering.cpp.
References llvm::ISD::ABS, llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, CC, combineVectorSizedSetCCEquality(), llvm::SelectionDAG::computeKnownBits(), DL, llvm::ISD::FNEG, llvm::SelectionDAG::getAllOnesConstant(), llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNegative(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCSwappedOperands(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasOneUse(), llvm::Value::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), incDecVectorConstant(), llvm::KnownBits::intersectWith(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllZeros(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::EVT::isInteger(), llvm::APInt::isMinSignedValue(), isNullFPScalarOrVectorConst(), llvm::TargetLoweringBase::isOperationLegal(), llvm::APInt::isPowerOf2(), llvm::EVT::isScalarInteger(), llvm::APInt::isSignBitSet(), llvm::ISD::isSignedIntSetCC(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::isUnsignedIntSetCC(), llvm::EVT::isVector(), LHS, LowerVSETCC(), llvm::SelectionDAG::MaskedValueIsZero(), MatchVectorAllEqualTest(), N, llvm::KnownBits::One, llvm::ISD::OR, RHS, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, truncateAVX512SetCCNoBWI(), llvm::ISD::UMIN, and llvm::KnownBits::Zero.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction.
Note that this is only legal for some op/cc combinations.
Definition at line 46449 of file X86ISelLowering.cpp.
References llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_SUB, CC, llvm::X86ISD::CMP, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NS, llvm::X86::COND_S, llvm::SelectionDAG::getAtomic(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), lowerAtomicArithWithLOCK(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and llvm::X86ISD::SUB.
Referenced by combineSetCCEFLAGS().
|
static |
Optimize an EFLAGS definition used according to the condition code CC
into a simpler EFLAGS value, potentially returning a new CC
and replacing uses of chain values.
Definition at line 47254 of file X86ISelLowering.cpp.
References CC, checkBoolTestSetCCCombine(), checkSignTestSetCCCombine(), combineCarryThroughADD(), combinePTESTCC(), combineSetCCAtomicArith(), combineSetCCMOVMSK(), and llvm::X86::COND_B.
Referenced by combineBrCond(), combineCMov(), and combineX86SetCC().
|
static |
Definition at line 47019 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, canScaleShuffleElements(), CC, llvm::MVT::changeVectorElementTypeToInteger(), llvm::X86ISD::CMP, collectConcatOps(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getSplitVectorSrc(), getTargetShuffleInputs(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), isCompletePermute(), llvm::APInt::isMask(), llvm::SDValue::isUndef(), llvm::APInt::isZero(), LHS, llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PCMPEQ, llvm::peekThroughBitcasts(), llvm::X86ISD::PTEST, RHS, llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::SUB, llvm::X86ISD::TESTP, llvm::ISD::TRUNCATE, and llvm::ISD::XOR.
Referenced by combineSetCCEFLAGS().
|
static |
Definition at line 53962 of file X86ISelLowering.cpp.
References combineExtSetcc(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineToExtendBoolVectorInReg(), combineToExtendCMOV(), DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isVector(), N, promoteExtBeforeAdd(), PromoteMaskArithmetic(), llvm::X86ISD::SETCC_CARRY, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, and llvm::ISD::TRUNCATE.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 53688 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::X86ISD::CMOV, DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), N, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSignExtendInReg().
|
static |
Definition at line 48138 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, Cond, llvm::ISD::Constant, DL, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::EVT::isInteger(), llvm::EVT::isVector(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), N, llvm::X86ISD::SETCC_CARRY, llvm::ISD::SETUGE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, supportedVectorVarShift(), llvm::ISD::VSELECT, llvm::X86ISD::VSHLV, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 48206 of file X86ISelLowering.cpp.
References combineShiftToPMULH(), llvm::ISD::Constant, DL, llvm::APInt::eq(), llvm::SDNode::getAsAPIntVal(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDValue::hasOneUse(), llvm::APInt::isNegative(), llvm::EVT::isVector(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_UMin(), llvm::PatternMatch::m_Value(), N, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, Size, llvm::ISD::SRA, supportedVectorVarShift(), llvm::APInt::ult(), and llvm::X86ISD::VSRAV.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 48274 of file X86ISelLowering.cpp.
References llvm::ISD::AND, combineShiftToPMULH(), Cond, llvm::APInt::countr_one(), DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignificantBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::APInt::isMask(), llvm::isPowerOf2_32(), llvm::APInt::lshr(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), N, llvm::ISD::SETUGE, llvm::ISD::SETULT, llvm::ISD::SRL, supportedVectorVarShift(), llvm::ISD::VSELECT, and llvm::X86ISD::VSRLV.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 48089 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), LHS, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, N, RHS, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.
Referenced by combineShiftRightArithmetic(), and combineShiftRightLogical().
|
static |
Definition at line 41780 of file X86ISelLowering.cpp.
References canonicalizeShuffleWithOp(), combineShuffleOfConcatUndef(), combineShuffleToAddSubOrFMAddSub(), combineTargetShuffle(), combineToConsecutiveLoads(), combineX86ShufflesRecursively(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorNumElements(), isSoftF16(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), N, narrowShuffle(), and llvm::TargetLowering::SimplifyDemandedVectorElts().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 41705 of file X86ISelLowering.cpp.
References Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::SDValue::isUndef(), and N.
Referenced by combineShuffle().
|
static |
Try to combine a shuffle into a target-specific add-sub or mul-add-sub node.
Definition at line 41664 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADDSUB, combineShuffleToFMAddSub(), DL, llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMSUBADD, llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::is512BitVector(), isAddSubOrSubAdd(), isFMAddSubOrFMSubAdd(), and N.
Referenced by combineShuffle().
|
static |
Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.
Definition at line 41622 of file X86ISelLowering.cpp.
References DL, llvm::ISD::FMA, llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMSUB, llvm::X86ISD::FMSUBADD, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDValue::hasOneUse(), isAddSubOrSubAddMask(), llvm::TargetLoweringBase::isTypeLegal(), N, std::swap(), and llvm::ISD::VECTOR_SHUFFLE.
Referenced by combineShuffleToAddSubOrFMAddSub().
|
static |
Definition at line 53749 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), combineSextInRegCmov(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::X86Subtarget::hasInt256(), llvm::ISD::isNormalLoad(), llvm::ISD::LOAD, N, PromoteMaskArithmetic(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::SIGN_EXTEND_INREG.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 55017 of file X86ISelLowering.cpp.
References assert(), llvm::BitWidth, llvm::X86TargetLowering::BuildFILD(), llvm::EVT::changeVectorElementType(), combineToFPTruncExtElt(), combineVectorCompareAndMaskUnaryOp(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::X86ISD::CVTSI2P, llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::X86Subtarget::getTargetLowering(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isNormalLoad(), llvm::MemSDNode::isSimple(), llvm::EVT::isVector(), llvm::ISD::LOAD, N, P, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SINT_TO_FP, llvm::X86ISD::STRICT_CVTSI2P, llvm::ISD::STRICT_SINT_TO_FP, and llvm::ISD::TRUNCATE.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 51638 of file X86ISelLowering.cpp.
References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::ISD::ANY_EXTEND, combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, detectSSatPattern(), detectUSatPattern(), EmitTruncSStore(), llvm::ISD::EXTRACT_VECTOR_ELT, F, llvm::CallingConv::Fast, llvm::MemSDNode::getAAInfo(), llvm::MemSDNode::getAddressSpace(), llvm::SelectionDAG::getAddrSpaceCast(), llvm::MemSDNode::getAlign(), llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::TypeSize::getFixed(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::EVT::getStoreSize(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getTruncStore(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::MachinePointerInfo::getWithOffset(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::Hi, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MemSDNode::isNonTemporal(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::MemSDNode::isSimple(), llvm::EVT::isSimple(), llvm::StoreSDNode::isTruncatingStore(), llvm::TargetLoweringBase::isTruncStoreLegal(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::Lo, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::SDNode::ops(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PEXTRW, llvm::X86AS::PTR32_SPTR, llvm::X86AS::PTR32_UPTR, llvm::X86AS::PTR64, llvm::ISD::SCALAR_TO_VECTOR, scalarizeVectorStore(), splitVectorStore(), llvm::ISD::TokenFactor, llvm::ISD::TRUNCATE, llvm::Align::value(), llvm::X86ISD::VTRUNC, llvm::X86ISD::VTRUNCS, and llvm::X86ISD::VTRUNCUS.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 55990 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::ISD::ADD, assert(), llvm::CallingConv::C, combineAddOrSubToADCOrSBB(), combineSubABS(), combineSubSetcc(), combineToHorizontalAddSub(), combineXorSubCTLZ(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getVTList(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDNode::hasOneUse(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::isNullConstant(), llvm::X86::isZeroNode(), N, llvm::X86ISD::SBB, llvm::ISD::SUB, and llvm::ISD::XOR.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 55900 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), CC, llvm::X86ISD::CMOV, Cond, llvm::X86::COND_NS, llvm::X86::COND_S, DL, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), N, llvm::X86ISD::SUB, and X.
Referenced by combineSub().
|
static |
Definition at line 55937 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADD, CC, DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), N, llvm::X86ISD::SETCC, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSub().
|
static |
Try to combine x86 target specific shuffles.
Definition at line 40720 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::X86ISD::BLENDI, llvm::CallingConv::C, canonicalizeLaneShuffleWithRepeatedOps(), collectConcatOps(), combineCommutableSHUFP(), combineRedundantDWordShuffle(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), llvm::ISD::CONCAT_VECTORS, D, DL, extract128BitVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::ISD::FADD, llvm::ISD::FDIV, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::ConstantVector::get(), llvm::MemSDNode::getBasePtr(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::TypeSize::getFixed(), llvm::MVT::getFixedSizeInBits(), llvm::MachineMemOperand::getFlags(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::User::getOperand(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), getPSHUFShuffleMask(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleAndZeroables(), getTargetShuffleInputs(), getTargetShuffleMask(), llvm::EVT::getTypeForEVT(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::MachinePointerInfo::getWithOffset(), getZeroVector(), llvm::APInt::getZExtValue(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), llvm::Value::hasOneUse(), Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::X86ISD::INSERTPS, llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::MVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isEXTLoad(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::MemSDNode::isSimple(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), isUndefOrInRange(), llvm::ISD::isUNINDEXEDLoad(), llvm::VectorType::isValidElementType(), llvm::EVT::isVector(), LHS, llvm::SelectionDAG::MaskedValueIsZero(), llvm::X86::MaxShuffleCombineDepth, llvm::MachineMemOperand::MOLoad, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSS, N, narrowLoadToVZLoad(), llvm::narrowShuffleMaskElts(), llvm::Offset, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, Ptr, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::SCALAR_TO_VECTOR, llvm::APIntOps::ScaleBitMask(), scaleShuffleElements(), llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::SmallVectorBase< Size_T >::size(), Size, llvm::SM_SentinelUndef, llvm::ISD::SRL, std::swap(), llvm::ISD::TRUNCATE, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, llvm::X86ISD::VSRLI, llvm::X86ISD::VZEXT_LOAD, llvm::X86ISD::VZEXT_MOVL, and widenSubVector().
Referenced by combineShuffle().
|
static |
Definition at line 54662 of file X86ISelLowering.cpp.
References llvm::APInt::getAllOnes(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), N, and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 7178 of file X86ISelLowering.cpp.
References assert(), DL, EltsFromConsecutiveLoads(), getShuffleScalarElt(), llvm::EVT::getVectorNumElements(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by combineShuffle().
|
static |
Definition at line 45396 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::SmallVectorImpl< T >::append(), assert(), llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitsSet(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isScalarInteger(), llvm::EVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SETEQ, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSelect(), combineSext(), and combineZext().
|
static |
Definition at line 53871 of file X86ISelLowering.cpp.
References llvm::X86ISD::CMOV, DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), and llvm::ISD::SIGN_EXTEND.
Referenced by combineSext(), and combineZext().
|
static |
If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector element with a bitcast.
This tries to keep the sequence on XMM registers rather than moving between vector and GPRs.
Definition at line 54921 of file X86ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), N, and llvm::ISD::TRUNCATE.
Referenced by combineSIntToFP().
|
static |
Definition at line 52134 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SmallVectorBase< Size_T >::empty(), llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, isHorizontalBinOp(), LHS, N, RHS, SplitOpsAndApply(), llvm::ISD::SUB, and llvm::ISD::VECTOR_SHUFFLE.
Referenced by combineAdd(), combineFaddFsub(), and combineSub().
|
static |
Definition at line 52661 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, combinePMULH(), combineTruncatedArithmetic(), combineTruncateWithSat(), detectPMADDUBSW(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::ISD::LRINT, llvm::X86ISD::MMX_MOVD2W, and N.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen.
e.g. TRUNC( BINOP( X, Y ) ) --> BINOP( TRUNC( X ), TRUNC( Y ) ) TODO: This overlaps with the generic combiner's visitTRUNCATE. Remove anything that is guaranteed to be transformed by DAGCombiner.
Definition at line 52354 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::isBuildVectorOfConstantSDNodes(), IsFreeTruncation, llvm::TargetLoweringBase::isOperationLegal(), llvm::EVT::isVector(), llvm::ISD::MUL, N, llvm::ISD::OR, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by combineTruncate().
|
static |
Definition at line 51074 of file X86ISelLowering.cpp.
References assert(), llvm::EVT::changeVectorElementType(), llvm::ISD::CONCAT_VECTORS, detectSSatPattern(), detectUSatPattern(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is512BitVector(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, truncateVectorWithPACK(), llvm::X86Subtarget::useAVX512Regs(), llvm::X86ISD::VTRUNCS, and llvm::X86ISD::VTRUNCUS.
Referenced by combineTruncate().
|
static |
Definition at line 54953 of file X86ISelLowering.cpp.
References llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::EVT::isVector(), N, P, llvm::SelectionDAG::SignBitIsZero(), llvm::ISD::SINT_TO_FP, llvm::ISD::STRICT_SINT_TO_FP, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 56058 of file X86ISelLowering.cpp.
References assert(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getScalarSizeInBits(), getTargetConstantBitsFromNode(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZero(), I, LHS, N, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, Results, and RHS.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 54863 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::BuildVectorSDNode::isConstant(), llvm::EVT::isVector(), and N.
Referenced by combineSIntToFP().
|
static |
Definition at line 48645 of file X86ISelLowering.cpp.
References assert(), combineHorizOpWithShuffle(), DL, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorVT(), llvm::X86ISD::HADD, llvm::X86ISD::HSUB, llvm::SDValue::isUndef(), LHS, N, llvm::X86ISD::PSHUFD, RHS, and shouldUseHorizontalOp().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 48879 of file X86ISelLowering.cpp.
References assert(), combineX86ShufflesRecursively(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), Idx, llvm::ISD::INSERT_VECTOR_ELT, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::isNullConstant(), llvm::EVT::isSimple(), llvm::SDValue::isUndef(), N, llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::ISD::SCALAR_TO_VECTOR, and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 48494 of file X86ISelLowering.cpp.
References assert(), combineHorizOpWithShuffle(), combineX86ShufflesRecursively(), llvm::SelectionDAG::ComputeNumSignBits(), Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), getConstVector(), getEXTEND_VECTOR_INREG(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX512(), llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::APInt::isIntN(), llvm::APInt::isNegative(), IsNOT(), llvm::SDValue::isUndef(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::APInt::setBit(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::APInt::trunc(), llvm::ISD::TRUNCATE, llvm::APInt::truncSSat(), llvm::X86ISD::VTRUNC, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 48726 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::APInt::ashrInPlace(), assert(), llvm::CallingConv::C, combineX86ShufflesRecursively(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), getConstVector(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getV4X86ShuffleImm(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::ISD::isBitwiseLogicOp(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDNode::isOnlyUserOf(), llvm::SDValue::isUndef(), LHS, llvm::APInt::lshrInPlace(), N, llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PSHUFD, RHS, llvm::TargetLowering::SimplifyDemandedBits(), llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and X.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 48693 of file X86ISelLowering.cpp.
References assert(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::EVT::getSimpleVT(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorAllZeros(), N, llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::X86ISD::VSHL, llvm::X86ISD::VSRA, and llvm::X86ISD::VSRL.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Try to map a 128-bit or larger integer comparison to vector instructions before type legalization splits it up into chunks.
Definition at line 22068 of file X86ISelLowering.cpp.
References assert(), CC, llvm::X86::COND_E, llvm::X86::COND_NE, DL, emitOrXorXorTree(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getScalarValueSizeInBits(), getSETCC(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::X86Subtarget::hasAVX(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::INSERT_SUBVECTOR, llvm::isNullConstant(), isOrXorXorTree(), llvm::EVT::isScalarInteger(), llvm::ISD::LOAD, llvm::X86ISD::MOVMSK, llvm::peekThroughBitcasts(), llvm::X86ISD::PTEST, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::TRUNCATE, llvm::X86Subtarget::useAVX512Regs(), X, llvm::ISD::XOR, Y, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSetCC().
|
static |
Definition at line 51910 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::DELETED_NODE, llvm::APInt::getLowBitsSet(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), N, and llvm::TargetLowering::SimplifyDemandedVectorElts().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 44530 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, createVPDPBUSD(), detectExtMul(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::isPowerOf2_32(), LHS, llvm::Log2_32(), llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::MUL, and RHS.
Referenced by combineExtractVectorElt().
|
static |
Definition at line 57390 of file X86ISelLowering.cpp.
References assert(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getScalarSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::Hi, I, llvm::ISD::isBuildVectorAllZeros(), LHS, llvm::Lo, N, RHS, llvm::APInt::sext(), llvm::TargetLowering::SimplifyDemandedVectorElts(), llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::VPMADDUBSW, llvm::X86ISD::VPMADDWD, and llvm::APInt::zext().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element.
This function will also call SimplifyDemandedBits on already created BLENDV to perform additional simplifications.
Definition at line 45706 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::BitWidth, llvm::X86ISD::BLENDV, llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), Cond, DL, llvm::SelectionDAG::getNode(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is128BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::TargetLowering::SimplifyDemandedBits(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), and llvm::ISD::VSELECT.
Referenced by combineSelect().
|
static |
If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation.
TODO: Move to DAGCombiner, possibly using TargetLowering::hasAndNot()?
Definition at line 45496 of file X86ISelLowering.cpp.
References llvm::And, llvm::ISD::AND, llvm::X86ISD::ANDNP, assert(), CC, llvm::SelectionDAG::ComputeNumSignBits(), Cond, DL, llvm::get(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LHS, N, llvm::Or, llvm::ISD::OR, RHS, llvm::ISD::SETCC, std::swap(), and llvm::ISD::VSELECT.
Referenced by combineSelect().
|
static |
Definition at line 52699 of file X86ISelLowering.cpp.
References detectSSatPattern(), detectUSatPattern(), DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), N, llvm::TargetLowering::SimplifyDemandedBits(), llvm::X86ISD::VTRUNCS, and llvm::X86ISD::VTRUNCUS.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 43657 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), Idx, and llvm::ISD::isBuildVectorOfConstantSDNodes().
Referenced by combineBitcast(), combineCastedMaskArithmetic(), combineSelect(), and combineStore().
|
static |
Definition at line 55347 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::ADD, assert(), combineAddOrSubToADCOrSBB(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineX86SubCmpForFlags(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getVTList(), llvm::isOneConstant(), LHS, N, RHS, llvm::ISD::SUB, and llvm::X86ISD::SUB.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 55964 of file X86ISelLowering.cpp.
References llvm::X86::COND_NE, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::isZeroNode(), N, llvm::X86ISD::SETCC, and llvm::X86ISD::SUB.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 54677 of file X86ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), llvm::ISD::DELETED_NODE, llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), N, and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 53428 of file X86ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::ISD::isNormalLoad(), N, narrowLoadToVZLoad(), llvm::TargetLowering::DAGCombinerInfo::recursivelyDeleteUnusedNodes(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and llvm::TargetLowering::SimplifyDemandedVectorElts().
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 54830 of file X86ISelLowering.cpp.
References CC, combineSetCCEFLAGS(), DL, getSETCC(), and N.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Combine an arbitrary chain of shuffles into a single instruction if possible.
This is the leaf of the recursive combine below. When we have found some chain of single-use x86 shuffle instructions and accumulated the combined shuffle mask represented by them, this will try to pattern match that mask into either a single instruction if there is a special purpose instruction for this operation, or into a PSHUFB instruction which is a fully general instruction but should only be used to replace chains over a certain depth.
Definition at line 38528 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::AllOnes, llvm::ISD::AND, assert(), combineX86ShuffleChainWithExtract(), llvm::ISD::CONCAT_VECTORS, llvm::Depth, DL, llvm::ArrayRef< T >::empty(), extract128BitVector(), extractSubVector(), llvm::X86ISD::EXTRQI, llvm::X86ISD::FAND, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::X86ISD::INSERTPS, llvm::X86ISD::INSERTQI, insertSubVector(), is128BitLaneCrossingShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isAnyZero(), llvm::MVT::isFloatingPoint(), isInRange(), isSequentialOrUndefInRange(), isSequentialOrUndefOrZeroInRange(), llvm::SelectionDAG::isSplatValue(), isTargetShuffleEquivalent(), llvm::TargetLoweringBase::isTypeLegal(), isUndef(), isUndefOrEqual(), isUndefOrInRange(), isUndefOrZero(), isUndefOrZeroInRange(), llvm::MVT::isVector(), LHS, llvm::Lo, lowerShuffleWithPERMV(), matchBinaryPermuteShuffle(), matchBinaryShuffle(), matchShuffleAsEXTRQ(), matchShuffleAsInsertPS(), matchShuffleAsINSERTQ(), matchShuffleAsVTRUNC(), matchUnaryPermuteShuffle(), matchUnaryShuffle(), llvm::X86::mayFoldLoad(), llvm::narrowShuffleMaskElts(), OpIndex, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveZeroablesFromTargetShuffle(), RHS, llvm::ISD::SCALAR_TO_VECTOR, scaleShuffleElements(), llvm::APInt::setBit(), llvm::SelectionDAG::shouldOptForSize(), llvm::X86ISD::SHUF128, llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::X86Subtarget::useAVX512Regs(), llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPPERM, llvm::ISD::VSELECT, llvm::X86ISD::VTRUNC, and widenSubVector().
Referenced by combineX86ShuffleChainWithExtract(), and combineX86ShufflesRecursively().
|
static |
Definition at line 39275 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), canonicalizeShuffleMaskWithCommute(), canWidenShuffleElements(), combineX86ShuffleChain(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::Depth, llvm::SmallVectorBase< Size_T >::empty(), llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), I, Idx, llvm::TargetLoweringBase::isTypeLegal(), llvm::peekThroughBitcasts(), resolveTargetShuffleInputsAndMask(), llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, and std::swap().
Referenced by combineX86ShuffleChain(), and combineX86ShufflesRecursively().
|
static |
Definition at line 39606 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), getZeroVector(), I, llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::none_of(), llvm::APInt::setBit(), llvm::SelectionDAG::shouldOptForSize(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineTargetShuffle(), and combineX86ShufflesRecursively().
|
static |
Fully generic combining of x86 shuffle instructions.
This should be the last combine run over the x86 shuffle instructions. Once they have been fully optimized, this will recursively consider all chains of single-use shuffle instructions, build a generic model of the cumulative shuffle operation, and check for simpler instructions which implement this operation. We use this primarily for two purposes:
1) Collapse generic shuffles to specialized single instructions when equivalent. In most cases, this is just an encoding size win, but sometimes we will collapse multiple generic shuffles into a single special-purpose shuffle. 2) Look for sequences of shuffle instructions with 3 or more total instructions, and replace them with the slightly more expensive SSSE3 PSHUFB instruction if available. We do this as the last combining step to ensure we avoid using PSHUFB if we can implement the shuffle with a suitable short sequence of other instructions. The PSHUFB will either use a register or have to read from memory and so is slightly (but only slightly) more expensive than the other shuffle instructions.
Because this is inherently a quadratic operation (for each shuffle in a chain, we recurse up the chain), the depth is limited to 8 instructions. This should never be an issue in practice as the shuffle lowering doesn't produce sequences of more than 8 instructions.
FIXME: We will currently miss some cases where the redundant shuffling would simplify under the threshold for PSHUFB formation because of combine-ordering. To fix this, we should do the redundant instruction combining in this recursive walk.
Definition at line 39728 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::any_of(), llvm::SmallVectorImpl< T >::append(), llvm::SDNode::areOnlyUsersOf(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), canonicalizeShuffleMaskWithCommute(), canonicalizeShuffleMaskWithHorizOp(), canWidenShuffleElements(), combineX86ShuffleChain(), combineX86ShuffleChainWithExtract(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::countr_zero(), llvm::Depth, DL, llvm::ArrayRef< T >::empty(), llvm::SmallVectorBase< Size_T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::enumerate(), llvm::ISD::EXTRACT_SUBVECTOR, extractBits(), llvm::APInt::extractBits(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::SelectionDAG::getBitcast(), getNode(), getOnesVector(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::Hi, I, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::is_contained(), llvm::APInt::isAllOnes(), llvm::ISD::isBuildVectorAllOnes(), isInRange(), llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::EVT::isSimple(), isTargetShuffleVariableMask(), isUndefOrZero(), llvm::EVT::isVector(), llvm::MVT::isVector(), llvm::APInt::isZero(), llvm::KnownBits::isZero(), LHS, llvm::Lo, MaxDepth, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveTargetShuffleFromZeroables(), resolveTargetShuffleInputsAndMask(), RHS, llvm::APIntOps::ScaleBitMask(), llvm::APInt::setBit(), llvm::TargetLowering::SimplifyMultipleUseDemandedVectorElts(), llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, std::swap(), llvm::APInt::trunc(), widenSubVector(), and llvm::APInt::zext().
Referenced by combineAnd(), combineAndnp(), combineEXTEND_VECTOR_INREG(), combineINSERT_SUBVECTOR(), combineOr(), combineShuffle(), combineTargetShuffle(), combineVectorInsert(), combineVectorPack(), combineVectorShiftImm(), combineX86ShufflesRecursively(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
|
static |
Helper entry wrapper to combineX86ShufflesRecursively.
Definition at line 40165 of file X86ISelLowering.cpp.
References combineX86ShufflesRecursively(), and llvm::X86::MaxShuffleCombineDepth.
|
static |
Definition at line 49746 of file X86ISelLowering.cpp.
References llvm::X86ISD::BRCOND, CC, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::X86::COND_NE, llvm::SDNode::getAsAPIntVal(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::X86::GetOppositeBranchCondition(), llvm::APInt::getSExtValue(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), llvm::isNullConstant(), llvm::isOneConstant(), llvm_unreachable, N, llvm::SDNode::op_values(), llvm::X86ISD::SETCC, llvm::X86ISD::SUB, and X.
Referenced by combineCMP(), and combineX86AddSub().
|
static |
Definition at line 53078 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineBMILogicOp(), combineFneg(), combineOrXorWithSETCC(), combineXorSubCTLZ(), convertIntLogicToFPLogic(), DL, foldVectorXorShiftIntoCmp(), foldXor1SetCC(), foldXorTruncShiftIntoCmp(), llvm::X86ISD::FXOR, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllOnes(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), LHS, N, RHS, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 53022 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BSR, llvm::CallingConv::C, llvm::ISD::CTLZ_ZERO_UNDEF, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), N, llvm::ISD::SUB, llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSub(), and combineXor().
|
static |
Definition at line 54177 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, combineExtSetcc(), combineOrCmpEqZeroToCtlzSrl(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), combineToExtendBoolVectorInReg(), combineToExtendCMOV(), concatSubVectors(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::X86ISD::PACKUS, promoteExtBeforeAdd(), PromoteMaskArithmetic(), llvm::X86ISD::SETCC_CARRY, and llvm::ISD::TRUNCATE.
Referenced by llvm::X86TargetLowering::PerformDAGCombine().
|
static |
Definition at line 45860 of file X86ISelLowering.cpp.
References canCombineAsMaskOperation(), Cond, DL, llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::X86Subtarget::hasAVX512(), LHS, N, RHS, llvm::ISD::SETCC, and llvm::ISD::VSELECT.
Referenced by combineSelect().
|
static |
Helper to get compute inlane shuffle mask for a complete shuffle mask.
Definition at line 14861 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::assign(), and Size.
Referenced by lowerShuffleAsLanePermuteAndShuffle(), and lowerV8F32Shuffle().
|
static |
Definition at line 37200 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::Depth, llvm::getHorizDemandedEltsForFirstOperand(), and llvm::APInt::isZero().
|
static |
Definition at line 37177 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::Depth, llvm::APInt::getSplat(), llvm::Hi, LHS, llvm::Lo, llvm::KnownBits::mul(), RHS, llvm::KnownBits::sadd_sat(), llvm::APIntOps::ScaleBitMask(), llvm::KnownBits::sext(), and llvm::KnownBits::zext().
Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode().
|
static |
Definition at line 37154 of file X86ISelLowering.cpp.
References llvm::KnownBits::computeForAddSub(), llvm::SelectionDAG::computeKnownBits(), llvm::Depth, llvm::APInt::getSplat(), llvm::Hi, LHS, llvm::Lo, llvm::KnownBits::mul(), RHS, llvm::APIntOps::ScaleBitMask(), and llvm::KnownBits::sext().
Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode().
|
static |
Definition at line 37134 of file X86ISelLowering.cpp.
References llvm::KnownBits::abdu(), llvm::KnownBits::computeForAddSub(), llvm::SelectionDAG::computeKnownBits(), llvm::Depth, LHS, RHS, llvm::APIntOps::ScaleBitMask(), and llvm::KnownBits::zext().
Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode().
|
static |
Compute whether each element of a shuffle is zeroable.
A "zeroable" vector shuffle element is one which can be lowered to zero. Either it is an undef element in the shuffle mask, the element of the input referenced is undef, or the element of the input referenced is known to be zero. Many x86 shuffles can zero lanes cheaply and we often want to handle as many lanes with this technique as possible to simplify the remaining shuffle.
Definition at line 5548 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::APInt::extractBits(), llvm::SDValue::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::APInt::getZero(), llvm::ISD::isBuildVectorAllZeros(), llvm::X86::isZeroNode(), llvm::peekThroughBitcasts(), llvm::APInt::setBit(), and Size.
Referenced by lowerVECTOR_SHUFFLE().
|
static |
Definition at line 4490 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getContext(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), and insertSubVector().
Referenced by combineConcatVectorOps(), combineZext(), EltsFromConsecutiveLoads(), and lowerV8I16Shuffle().
|
static |
If both input operands of a logic op are being cast from floating-point types or FP compares, try to convert this into a floating-point logic node to avoid unnecessary moves from SSE to integer registers.
Definition at line 49240 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, cheapX86FSETCC_SSE(), convertIntLogicToFPLogicOpcode(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, llvm::ISD::SCALAR_TO_VECTOR, and llvm::ISD::SETCC.
Referenced by combineAnd(), combineOr(), and combineXor().
Definition at line 49224 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm_unreachable, llvm::ISD::OR, and llvm::ISD::XOR.
Referenced by combineBitOpWithMOVMSK(), and convertIntLogicToFPLogic().
|
static |
Definition at line 29410 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, convertShiftLeftToScale(), llvm::ISD::FP_TO_SINT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPack(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, I, llvm::Lo, llvm::X86ISD::PACKUS, llvm::APInt::shl(), and llvm::ISD::SHL.
Referenced by convertShiftLeftToScale(), LowerRotate(), and LowerShift().
|
static |
Definition at line 43730 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::append(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumOperands(), llvm::SDNode::getOperand(), llvm::TargetLoweringBase::getPointerTy(), llvm::BuildVectorSDNode::getSplatValue(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasSSE1(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::X86ISD::MMX_MOVW2D, llvm::X86ISD::MOVDQ2Q, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SmallVectorBase< Size_T >::size(), and llvm::Splat.
Referenced by combineBitcast().
|
static |
Create a shuffle mask that matches the PACKSS/PACKUS truncation.
A multi-stage pack shuffle mask is created by specifying NumStages > 1. Note: This ignores saturation, so inputs must be checked first.
Definition at line 5164 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), and llvm::Offset.
Referenced by getFauxShuffleMask(), and matchShuffleWithPACK().
|
static |
Definition at line 34910 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), CC, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::contains(), llvm::TargetSubtargetInfo::getInstrInfo(), llvm::X86::GetOppositeBranchCondition(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getSubtarget(), std::swap(), and TII.
|
static |
Definition at line 44287 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), llvm::X86ISD::PSADBW, RegSize, and SplitOpsAndApply().
Referenced by combineBasicSADPattern().
|
static |
Definition at line 27847 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::X86ISD::FLDENVm, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasSSE1(), llvm::ISD::INTRINSIC_VOID, Ptr, and X87StateSize.
|
static |
Definition at line 5766 of file X86ISelLowering.cpp.
References Cond, llvm::EVT::getScalarSizeInBits(), getTargetConstantBitsFromNode(), llvm::EVT::getVectorNumElements(), isZero(), and llvm::SM_SentinelUndef.
Referenced by combineSelect(), getFauxShuffleMask(), and lowerVSELECTtoVectorShuffle().
|
static |
Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source values and a vector of extraction indices.
The vectors might be manipulated to match the type of the permute op.
Definition at line 8459 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::EVT::changeVectorElementTypeToInteger(), createVariablePermute(), DL, extractSubVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getSelectCC(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), Idx, llvm::isPowerOf2_64(), llvm::ISD::MUL, llvm::X86ISD::PSHUFB, llvm::ISD::SETEQ, llvm::MVT::SimpleTy, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, widenSubVector(), and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by createVariablePermute(), and LowerBUILD_VECTORAsVariablePermute().
|
static |
Definition at line 44241 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), LHS, RegSize, RHS, SplitOpsAndApply(), and llvm::X86ISD::VPDPBUSD.
Referenced by combineVPDPBUSDPattern().
|
static |
Definition at line 44190 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::KnownBits::countMaxActiveBits(), llvm::SDValue::getOpcode(), llvm::BuildVectorSDNode::isConstant(), IsFreeTruncation, llvm::Mul, llvm::ISD::SIGN_EXTEND, std::swap(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineVPDPBUSDPattern().
|
static |
Definition at line 52519 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::ISD::BUILD_VECTOR, detectSSatPattern(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSSE3(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::ISD::MUL, llvm::ISD::SIGN_EXTEND, SplitOpsAndApply(), std::swap(), llvm::X86ISD::VPMADDUBSW, and llvm::ISD::ZERO_EXTEND.
Referenced by combineTruncate().
Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type).
With MatchPackUS, the smax/smin range is [0, unsigned_max_of_dest_type]. Return the source value to be truncated or SDValue() if the pattern was not matched.
Definition at line 51040 of file X86ISelLowering.cpp.
References assert(), llvm::CallingConv::C, llvm::APInt::getAllOnes(), llvm::EVT::getScalarSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::ISD::isConstantSplatVector(), llvm::APInt::sext(), llvm::SMax, llvm::ISD::SMAX, llvm::SMin, llvm::ISD::SMIN, and llvm::APInt::zext().
Referenced by combineStore(), combineTruncateWithSat(), combineVTRUNC(), and detectPMADDUBSW().
|
static |
Detect patterns of truncation with unsigned saturation:
(truncate (smin (smax (x, C1), C2)) to dest_type), where C1 >= 0 and C2 is unsigned max of destination type.
(truncate (smax (smin (x, C2), C1)) to dest_type) where C1 >= 0, C2 is unsigned max of destination type and C1 <= C2.
These two patterns are equivalent to: (truncate (umin (smax(x, C1), unsigned_max_of_dest_type)) to dest_type) So return the smax(x, C1) value to be truncated or SDValue() if the pattern was not matched.
Definition at line 50993 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::ISD::isConstantSplatVector(), llvm::APInt::isMask(), llvm::APInt::isNonNegative(), llvm::SMax, llvm::ISD::SMAX, llvm::SMin, llvm::ISD::SMIN, llvm::APInt::uge(), llvm::UMin, and llvm::ISD::UMIN.
Referenced by combineStore(), combineTruncateWithSat(), and combineVTRUNC().
Definition at line 44223 of file X86ISelLowering.cpp.
References llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::SUB, and llvm::ISD::ZERO_EXTEND.
Referenced by combineBasicSADPattern().
|
static |
Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'.
Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a
Definition at line 6896 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::sampleprof::Base, concatSubVectors(), llvm::APInt::countr_zero(), DL, llvm::ArrayRef< T >::drop_back(), EltsFromConsecutiveLoads(), llvm::APInt::extractBits(), findEltLoadSrc(), llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::MemSDNode::getAlign(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MachineMemOperand::getFlags(), llvm::EVT::getFloatingPointVT(), llvm::MVT::getFloatingPointVT(), llvm::EVT::getIntegerVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::EVT::getStoreSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::APInt::isAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::MachinePointerInfo::isDereferenceable(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::MemSDNode::isNonTemporal(), llvm::TargetLoweringBase::isOperationLegal(), llvm::isPowerOf2_32(), llvm::MemSDNode::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), isUndef(), llvm::EVT::isVector(), llvm::APInt::isZero(), llvm::X86::isZeroNode(), llvm::ISD::LOAD, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::Match, llvm::X86::mayFoldLoadIntoBroadcastFromMem(), llvm::MachineMemOperand::MOLoad, llvm::Offset, llvm::peekThroughBitcasts(), llvm::APInt::popcount(), llvm::APInt::setBit(), llvm::ArrayRef< T >::size(), llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VZEXT_LOAD.
Referenced by combineConcatVectorOps(), combineToConsecutiveLoads(), and EltsFromConsecutiveLoads().
|
static |
Definition at line 23697 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, CC, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::isAllOnesConstant(), llvm::isNullConstant(), llvm::X86ISD::KORTEST, llvm::X86ISD::KTEST, LHS, llvm::ISD::OR, RHS, llvm::ISD::SETEQ, and llvm::ISD::SETNE.
|
static |
Emit nodes that will be selected as "cmp Op0,Op1", or something equivalent.
Definition at line 22706 of file X86ISelLowering.cpp.
References llvm::Add, llvm::X86ISD::ADD, assert(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::X86::COND_E, llvm::X86::COND_NE, EmitTest(), llvm::MachineFunction::getFunction(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::Function::hasMinSize(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), isX86CCSigned(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::X86::mayFoldLoad(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::X86ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
|
static |
Emit a locked operation on a stack location which does not change any memory location, but does involve a lock prefix.
Location is chosen to be a) very likely accessed only by a single thread to minimize cache traffic, and b) definitely dereferenceable. Returns the new Chain result.
Definition at line 30998 of file X86ISelLowering.cpp.
References DL, llvm::X86Subtarget::getFrameLowering(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), and llvm::X86FrameLowering::has128ByteRedZone().
Referenced by LowerATOMIC_FENCE(), LowerATOMIC_STORE(), and lowerAtomicArith().
|
static |
Emit Masked Truncating Store with signed or unsigned saturation.
Definition at line 26760 of file X86ISelLowering.cpp.
References DL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getVTList(), Ptr, llvm::X86ISD::VMTRUNCSTORES, and llvm::X86ISD::VMTRUNCSTOREUS.
Referenced by LowerINTRINSIC_W_CHAIN().
|
static |
Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp expansion.
Definition at line 22041 of file X86ISelLowering.cpp.
References A, llvm::ISD::AND, B, DL, emitOrXorXorTree(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm_unreachable, llvm::ISD::OR, llvm::ISD::SETEQ, llvm::ISD::SETNE, X, and llvm::ISD::XOR.
Referenced by combineVectorSizedSetCCEquality(), and emitOrXorXorTree().
|
static |
Emit nodes that will be selected as "test Op0,Op0", or something equivalent.
Definition at line 22596 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86ISD::ADD, llvm::ISD::AND, llvm::X86ISD::AND, llvm::X86ISD::CMP, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NO, llvm::X86::COND_O, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), hasNonFlagsUse(), isProfitableToUseFlagOp(), llvm_unreachable, llvm::ISD::MUL, llvm::ISD::OR, llvm::X86ISD::OR, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SHL, llvm::ISD::SSUBO, llvm::ISD::SUB, llvm::X86ISD::SUB, llvm::ISD::USUBO, llvm::ISD::XOR, and llvm::X86ISD::XOR.
Referenced by EmitCmp().
|
static |
Emit Truncating Store with signed or unsigned saturation.
Definition at line 26749 of file X86ISelLowering.cpp.
References DL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVTList(), Ptr, llvm::X86ISD::VTRUNCSTORES, and llvm::X86ISD::VTRUNCSTOREUS.
Referenced by combineStore(), and LowerINTRINSIC_W_CHAIN().
|
static |
Utility function to emit xbegin specifying the start of an RTM region.
Definition at line 34489 of file X86ISelLowering.cpp.
References llvm::MachineInstrBuilder::addImm(), llvm::MachineBasicBlock::addLiveIn(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getBasicBlock(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, llvm::MachineFunction::insert(), isEFLAGSLiveAfter(), MBB, MI, MRI, llvm::MachineBasicBlock::splice(), TII, and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().
Referenced by llvm::X86TargetLowering::EmitInstrWithCustomInserter().
|
static |
Definition at line 20820 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::X86ISD::BLENDV, llvm::X86ISD::CVTTP2SI, llvm::ISD::FSUB, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::hasAVX2(), llvm::ISD::OR, and llvm::X86ISD::VSRAI.
Referenced by llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector.
This is a helper function of LowerToHorizontalOp(). This function expects two 256-bit vectors called V0 and V1. At first, each vector is split into two separate 128-bit vectors. Then, the resulting 128-bit vectors are used to implement two horizontal binary operations.
The kind of horizontal binary operation is defined by X86Opcode
.
Mode
specifies how the 128-bit parts of V0 and V1 are passed in input to the two new horizontal binop. When Mode is set, the first horizontal binop dag node would take as input the lower 128-bit of V0 and the upper 128-bit of V0. The second horizontal binop dag node would take as input the lower 128-bit of V1 and the upper 128-bit of V1. Example: HADD V0_LO, V0_HI HADD V1_LO, V1_HI
Otherwise, the first horizontal binop dag node takes as input the lower 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop dag node takes the upper 128-bit of V0 and the upper 128-bit of V1. Example: HADD V0_LO, V1_LO HADD V0_HI, V1_HI
If isUndefLO
is set, then the algorithm propagates UNDEF to the lower 128-bits of the result. If isUndefHI
is set, then UNDEF is propagated to the upper 128-bits of the result.
Definition at line 7870 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, extract128BitVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::is256BitVector(), and llvm::SDNode::isUndef().
Referenced by LowerToHorizontalOp().
|
static |
Handles the lowering of builtin intrinsics with chain that return their value into registers EDX:EAX.
If operand ScrReg is a valid register identifier, then operand 2 of N is copied to SrcReg. The assumption is that SrcReg is an implicit input to TargetOpcode. Returns a Glue value which can be used to add extra copy-from-reg if the expanded intrinsics implicitly defines extra registers (i.e. not just EDX:EAX).
Definition at line 26626 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_PAIR, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getMachineNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), N, llvm::ISD::OR, Results, and llvm::ISD::SHL.
Referenced by getReadTimeStampCounter(), LowerINTRINSIC_W_CHAIN(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Widen a vector input to a vector of NVT.
The input vector must have the same element type as NVT.
Definition at line 31917 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SDValue::isUndef(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by LowerMGATHER(), LowerMLOAD(), LowerMSCATTER(), and LowerMSTORE().
|
static |
Generate a DAG to grab 128-bits from a vector > 128 bits.
This sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128 or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4 instructions or a simple subregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering EXTRACT_VECTOR_ELT operations easier.
Definition at line 3951 of file X86ISelLowering.cpp.
References assert(), extractSubVector(), llvm::SDValue::getValueType(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().
Referenced by canonicalizeShuffleMaskWithHorizOp(), combineArithReduction(), combineExtractWithShuffle(), combineTargetShuffle(), combineX86ShuffleChain(), ExpandHorizontalBinOp(), getTargetVShiftNode(), lowerAddSubToHorizontalOp(), lowerShuffleAsBroadcast(), LowerTruncateVecI1(), lowerV8I16Shuffle(), narrowExtractedVectorSelect(), llvm::X86TargetLowering::SimplifyDemandedBitsForTargetNode(), and vectorizeExtractedCast().
|
static |
Generate a DAG to grab 256-bits from a 512-bit vector.
Definition at line 3959 of file X86ISelLowering.cpp.
References assert(), extractSubVector(), llvm::SDValue::getValueType(), and llvm::EVT::is512BitVector().
|
static |
Extract one bit from mask vector, like v16i1 or v8i1.
AVX-512 feature.
Definition at line 17941 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), Idx, llvm::X86ISD::KSHIFTR, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and widenMaskVector().
|
static |
Definition at line 3914 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::SDValue::isUndef(), and llvm::SDNode::ops().
|
static |
Definition at line 6849 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, findEltLoadSrc(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), Idx, llvm::ISD::isNON_EXTLoad(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SRL, and llvm::ISD::TRUNCATE.
Referenced by EltsFromConsecutiveLoads(), and findEltLoadSrc().
|
static |
Definition at line 30529 of file X86ISelLowering.cpp.
References llvm::CallingConv::C, ConstantBit, I, llvm::isPowerOf2_64(), llvm::PatternMatch::m_AllOnes(), llvm::PatternMatch::m_APInt(), llvm::PatternMatch::m_c_And(), llvm::PatternMatch::m_Not(), llvm::PatternMatch::m_Sub(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), NotConstantBit, NotShiftBit, ShiftBit, and UndefBit.
|
static |
Definition at line 57801 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), N, llvm::SDVTList::NumVTs, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SDVTList::VTs.
Referenced by combineINTRINSIC_VOID(), combineINTRINSIC_W_CHAIN(), and combineINTRINSIC_WO_CHAIN().
|
static |
Fold "masked merge" expressions like (m & x) | (~m & y)
into the equivalent ((x ^ y) & m) ^ y)
pattern.
This is typically a better representation for targets without a fused "and-not" operation. This function is intended to be called from a TargetLowering::PerformDAGCombine
callback on ISD::OR
nodes.
Definition at line 50435 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), DL, foldMaskedMergeImpl(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::hasOneUse(), and llvm::ISD::OR.
Referenced by combineOr().
|
static |
Definition at line 50409 of file X86ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::ISD::FREEZE, llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::isBitwiseNot(), std::swap(), and llvm::ISD::XOR.
Referenced by foldMaskedMerge().
|
static |
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1.
This should be called before type legalization because the pattern may not persist after that.
Definition at line 50937 of file X86ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), llvm::isConstOrConstSplat(), llvm::EVT::isSimple(), N, llvm::ISD::SETGT, llvm::MVT::SimpleTy, and llvm::ISD::SRA.
|
static |
Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
Definition at line 53008 of file X86ISelLowering.cpp.
References DL, llvm::X86::GetOppositeBranchCondition(), getSETCC(), llvm::isOneConstant(), LHS, N, llvm::X86ISD::SETCC, and llvm::ISD::XOR.
Referenced by combineXor().
|
static |
Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1)
Definition at line 50882 of file X86ISelLowering.cpp.
References Cond, DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::isOneConstant(), N, llvm::ISD::SETGT, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineXor().
Definition at line 43430 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::X86ISD::FAND, llvm::X86ISD::FANDN, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm_unreachable, llvm::ISD::OR, and llvm::ISD::XOR.
Referenced by adjustBitcastSrcVectorSSE1().
|
static |
Definition at line 26495 of file X86ISelLowering.cpp.
References llvm::sampleprof::Base, llvm::CallingConv::C, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::ISD::isBuildVectorAllOnes(), and llvm::X86ISD::MGATHER.
Referenced by LowerINTRINSIC_W_CHAIN().
|
static |
Definition at line 4261 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, DL, extractSubVector(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), llvm::peekThroughBitcasts(), Widen(), and widenSubVector().
Referenced by canonicalizeBitSelect(), and LowerFunnelShift().
|
static |
Definition at line 10097 of file X86ISelLowering.cpp.
References DL, extractSubVector(), getAVX512TruncNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::TRUNCATE, llvm::X86ISD::VTRUNC, and widenSubVector().
Referenced by getAVX512TruncNode(), lowerShuffleAsVTRUNC(), and lowerShuffleWithVPMOV().
|
static |
Definition at line 4539 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, DL, llvm::SelectionDAG::getNode(), LHS, llvm::ISD::OR, and RHS.
Referenced by lowerShuffleAsBitBlend().
|
static |
Definition at line 49682 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, llvm::Depth, DL, getBMIMatchingOp(), llvm::SelectionDAG::getNode(), llvm::isAllOnesConstant(), llvm::isNullConstant(), llvm::isOneConstant(), and llvm::ISD::SUB.
Referenced by combineBMILogicOp(), and getBMIMatchingOp().
|
static |
Definition at line 6372 of file X86ISelLowering.cpp.
References assert(), DL, llvm::MemSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::EVT::getStoreSize(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::MemSDNode::isNonTemporal(), llvm::MemSDNode::isSimple(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::Offset, Ptr, llvm::MemSDNode::readMem(), llvm::X86ISD::SUBV_BROADCAST_LOAD, and llvm::X86ISD::VBROADCAST_LOAD.
Referenced by combineConcatVectorOps(), combineINSERT_SUBVECTOR(), lowerV2X128Shuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
Helper for attempting to create a X86ISD::BT node.
Definition at line 21980 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::X86ISD::BT, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SelectionDAG::MaskedValueIsZero(), and llvm::ISD::TRUNCATE.
Referenced by combineAnd(), combineCarryThroughADD(), and LowerAndToBT().
|
static |
Definition at line 7195 of file X86ISelLowering.cpp.
References assert(), llvm::CallingConv::C, llvm::ConstantVector::get(), llvm::UndefValue::get(), llvm::Constant::getIntegerValue(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::EVT::getTypeForEVT(), I, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEhalf(), llvm::APFloatBase::IEEEsingle(), llvm::MVT::isFloatingPoint(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by combineConcatVectorOps(), and lowerBuildVectorAsBroadcast().
|
static |
Definition at line 7220 of file X86ISelLowering.cpp.
References assert(), llvm::CallingConv::C, llvm::APInt::extractBits(), llvm::ConstantVector::get(), llvm::Constant::getIntegerValue(), llvm::Type::getIntNTy(), llvm::MVT::getScalarSizeInBits(), I, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEhalf(), llvm::APFloatBase::IEEEsingle(), llvm::MVT::isFloatingPoint(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
|
static |
Definition at line 3816 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::append(), assert(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEsingle(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
|
static |
Definition at line 3857 of file X86ISelLowering.cpp.
References getConstVector(), and llvm::APInt::getZero().
|
static |
Definition at line 3786 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by combineAndnp(), combineConcatVectorOps(), combineSelect(), combineVectorCompare(), combineVectorPack(), combineVectorShiftImm(), combineVPMADD(), combineX86ShuffleChain(), combineX86ShufflesConstants(), getConstVector(), IsNOT(), lowerShuffleAsElementInsertion(), lowerShuffleWithPERMV(), lowerV16F32Shuffle(), lowerV16I8Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().
|
static |
Definition at line 4513 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), DL, extractSubVector(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::EVT::isVector(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by combineVectorPack(), lowerShuffleAsSpecificZeroOrAnyExtend(), and llvm::X86TargetLowering::ReplaceNodeResults().
Definition at line 17987 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, getExtractedDemandedElts(), llvm::User::getOperand(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZero(), N, llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, llvm::APIntOps::ScaleBitMask(), llvm::APInt::setAllBits(), and llvm::APInt::setBit().
Referenced by getExtractedDemandedElts().
|
static |
Definition at line 5805 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::ISD::ANY_EXTEND, llvm::ISD::ANY_EXTEND_VECTOR_INREG, llvm::any_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), llvm::ISD::BITCAST, llvm::X86ISD::BLENDV, llvm::SmallVectorImpl< T >::clear(), llvm::SelectionDAG::ComputeNumSignBits(), Cond, createPackShuffleMask(), createShuffleMaskFromVSELECT(), llvm::DecodeZeroExtendMask(), llvm::Depth, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::getAllOnes(), llvm::APInt::getBitWidth(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::EVT::getFixedSizeInBits(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::APInt::getHighBitsSet(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPackDemandedElts(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetShuffleInputs(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isBuildVectorAllZeros(), llvm::EVT::isByteSized(), isInRange(), llvm::isNullConstant(), llvm::EVT::isSimple(), llvm::SDValue::isUndef(), isUndefOrInRange(), llvm::EVT::isVector(), llvm::APInt::isZero(), llvm::X86::isZeroNode(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::narrowShuffleMaskElts(), llvm::Offset, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::TRUNCATE, llvm::X86ISD::VBROADCAST, llvm::ISD::VECTOR_SHUFFLE, llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::ISD::VSELECT, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, llvm::X86ISD::VTRUNC, llvm::ISD::ZERO_EXTEND, llvm::ISD::ZERO_EXTEND_VECTOR_INREG, and llvm::APInt::zextOrTrunc().
Referenced by getTargetShuffleInputs().
|
static |
Definition at line 32310 of file X86ISelLowering.cpp.
References DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), and llvm::X86ISD::SUB.
Referenced by llvm::X86TargetLowering::visitMaskedLoad(), and llvm::X86TargetLowering::visitMaskedStore().
|
static |
Definition at line 26527 of file X86ISelLowering.cpp.
References llvm::sampleprof::Base, llvm::CallingConv::C, llvm::SelectionDAG::getDataLayout(), getMaskNode(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::ISD::isBuildVectorAllOnes(), and llvm::X86ISD::MGATHER.
Referenced by LowerINTRINSIC_W_CHAIN().
Definition at line 29084 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITREVERSE, getGFNICtrlImm(), llvm_unreachable, llvm::ISD::ROTL, llvm::ISD::ROTR, llvm::ISD::SHL, llvm::ISD::SRA, and llvm::ISD::SRL.
Referenced by getGFNICtrlImm(), and getGFNICtrlMask().
SDValue getGFNICtrlMask | ( | unsigned | Opcode, |
SelectionDAG & | DAG, | ||
const SDLoc & | DL, | ||
MVT | VT, | ||
unsigned | Amt = 0 |
||
) |
Definition at line 29107 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getGFNICtrlImm(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by LowerBITREVERSE(), LowerRotate(), and LowerShiftByScalarImmediate().
|
static |
If the input shuffle mask results in a vector that is undefined in all upper or lower half elements and that mask accesses only 2 halves of the shuffle's operands, return true.
A mask of half the width with mask indexes adjusted to access the extracted halves of the original shuffle operands is returned in HalfMask. HalfIdx1 and HalfIdx2 return whether the upper or lower half of each input operand is accessed.
Definition at line 15233 of file X86ISelLowering.cpp.
References assert(), isUndefLowerHalf(), isUndefUpperHalf(), and llvm::ArrayRef< T >::size().
Referenced by lowerShuffleWithUndefHalf(), and narrowShuffle().
|
static |
Definition at line 8195 of file X86ISelLowering.cpp.
References llvm::APInt::clearBit(), DL, extractSubVector(), llvm::APInt::getAllOnes(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorNumElements(), insertSubVector(), llvm::MVT::is256BitVector(), llvm::SDValue::isUndef(), and llvm::APInt::lshr().
Referenced by LowerToHorizontalOp().
|
static |
Definition at line 5211 of file X86ISelLowering.cpp.
References llvm::getHorizDemandedEltsForFirstOperand(), and llvm::EVT::getSizeInBits().
Referenced by llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
|
static |
Definition at line 49487 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::sampleprof::Base, llvm::LoadSDNode::getBasePtr(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::LSBaseSDNode::isIndexed(), and llvm::ISD::SHL.
Referenced by combineAndLoadToBZHI().
|
static |
Definition at line 35637 of file X86ISelLowering.cpp.
References assert(), and llvm_unreachable.
|
static |
Definition at line 32256 of file X86ISelLowering.cpp.
References assert(), I, llvm::StringRef::npos, and llvm::dwarf::toString().
Referenced by llvm::X86TargetLowering::isInlineAsmTargetBranch().
|
static |
Definition at line 54019 of file X86ISelLowering.cpp.
References llvm::any_of(), assert(), llvm::ISD::BUILD_VECTOR, llvm::ISD::FMA, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::STRICT_FMA.
Referenced by combineFMA().
|
static |
Return Mask with the necessary casting or extending for Mask
according to MaskVT
when lowering masking intrinsics.
Definition at line 25330 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::bitsLE(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorVT(), llvm::Hi, llvm::isAllOnesConstant(), llvm::X86::isZeroNode(), llvm::Lo, and llvm::SelectionDAG::SplitScalar().
Referenced by getGatherNode(), getPrefetchNode(), getScatterNode(), getVectorMaskingNode(), LowerINTRINSIC_W_CHAIN(), and lowerShuffleToEXPAND().
|
static |
If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value.
In the case of a vector, the returned constant will not contain undefined elements even if the input parameter does. This makes it suitable to be used as a replacement operand with operations (eg, bitwise-and) where an undef should not propagate.
Definition at line 53247 of file X86ISelLowering.cpp.
References getZeroVector(), and isNullFPScalarOrVectorConst().
Referenced by combineFAnd(), and combineFAndn().
|
static |
Returns a vector of specified type with all bits set.
Always build ones vectors as <4 x i32>, <8 x i32> or <16 x i32>. Then bitcast to their original type, ensuring they get CSE'd.
Definition at line 4505 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), and llvm::EVT::is512BitVector().
Referenced by combineEXTRACT_SUBVECTOR(), combineX86ShufflesRecursively(), and materializeVectorConstant().
|
static |
If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element.
Otherwise, return -1.
Definition at line 51357 of file X86ISelLowering.cpp.
References llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), and llvm::EVT::getVectorNumElements().
Referenced by getParamsForOneTrueMaskedElt().
Definition at line 35623 of file X86ISelLowering.cpp.
References llvm_unreachable.
|
static |
Returns a node that packs the LHS + RHS nodes together at half width.
May return X86ISD::PACKSS/PACKUS, packing the top/bottom half. TODO: Add subvector splitting if/when we have a need for it.
Definition at line 4617 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeMaxSignificantBits(), llvm::KnownBits::countMaxActiveBits(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE41(), I, LHS, llvm::Offset, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.
Referenced by convertShiftLeftToScale(), LowerFunnelShift(), LowerMUL(), LowerRotate(), and LowervXi8MulWithUNPCK().
|
static |
Definition at line 5186 of file X86ISelLowering.cpp.
References llvm::APInt::getBitWidth(), llvm::EVT::getSizeInBits(), llvm::APInt::getZero(), and llvm::APInt::setBit().
Referenced by llvm::X86TargetLowering::computeKnownBitsForTargetNode(), llvm::X86TargetLowering::ComputeNumSignBitsForTargetNode(), getFauxShuffleMask(), llvm::X86TargetLowering::SimplifyDemandedBitsForTargetNode(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
|
static |
Given a masked memory load/store operation, return true if it has one mask bit set.
If it has one mask bit set, then also return the memory address of the scalar element to load/store, the vector index to insert/extract that scalar element, and the alignment for the scalar memory access.
Definition at line 51392 of file X86ISelLowering.cpp.
References Addr, llvm::commonAlignment(), llvm::MemSDNode::getBasePtr(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MaskedLoadStoreSDNode::getMask(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemoryVT(), getOneTrueElt(), llvm::MemSDNode::getOriginalAlign(), llvm::EVT::getStoreSize(), llvm::EVT::getVectorElementType(), and llvm::Offset.
Referenced by reduceMaskedLoadToScalarLoad(), and reduceMaskedStoreToScalarStore().
|
static |
Definition at line 31121 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPMOVMSKB(), llvm::X86Subtarget::hasInt256(), llvm::Hi, llvm::Lo, llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::ISD::SHL, llvm::SelectionDAG::SplitVector(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineBitcastvxi1(), combinePredicateReduction(), combinePTESTCC(), getPMOVMSKB(), and LowerBITCAST().
|
static |
Definition at line 26596 of file X86ISelLowering.cpp.
References llvm::sampleprof::Base, llvm::CallingConv::C, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), getMaskNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), and llvm::MVT::getVectorVT().
Referenced by LowerINTRINSIC_W_CHAIN().
|
static |
Get the PSHUF-style mask from PSHUF node.
This is a very minor wrapper around getTargetShuffleMask to easy forming v4 PSHUF-style masks that can be reused with such instructions.
Definition at line 40178 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), getTargetShuffleMask(), llvm_unreachable, N, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, and llvm::X86ISD::PSHUFLW.
Referenced by combineRedundantDWordShuffle(), and combineTargetShuffle().
|
static |
Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp).
This function is also used to custom lower READCYCLECOUNTER nodes.
Definition at line 26681 of file X86ISelLowering.cpp.
References DL, expandIntrinsicWChainHelper(), llvm::SelectionDAG::getCopyFromReg(), llvm::SDValue::getValue(), N, and Results.
Referenced by LowerINTRINSIC_W_CHAIN(), LowerREADCYCLECOUNTER(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Creates an SDNode for a predicated scalar operation.
Mask
, Op
, PreservedSrc
). The mask is coming as MVT::i8 and it should be transformed to MVT::v1i1 while lowering masking intrinsics. The main difference between ScalarMaskingNode and VectorMaskingNode is using "X86select" instead of "vselect". We just can't create the "vselect" node for a scalar instruction. Definition at line 25390 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::X86ISD::FSETCCM, llvm::X86ISD::FSETCCM_SAE, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getZeroVector(), llvm::SDValue::isUndef(), llvm::X86ISD::SELECTS, and llvm::X86ISD::VFPCLASSS.
|
static |
Try to get a scalar value for a specific element of a vector.
Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.
Definition at line 12173 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), Idx, llvm::MVT::isVector(), llvm::peekThroughBitcasts(), and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by lowerShuffleAsElementInsertion(), and lowerV2F64Shuffle().
|
static |
Definition at line 26565 of file X86ISelLowering.cpp.
References llvm::sampleprof::Base, llvm::CallingConv::C, llvm::SelectionDAG::getDataLayout(), getMaskNode(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), and llvm::X86ISD::MSCATTER.
Referenced by LowerINTRINSIC_W_CHAIN().
Definition at line 25416 of file X86ISelLowering.cpp.
References llvm::classifyEHPersonality(), llvm::Function::getPersonalityFn(), llvm::Function::hasPersonalityFn(), llvm::MSVC_CXX, llvm::MSVC_X86SEH, and llvm::report_fatal_error().
Referenced by recoverFramePointer().
|
static |
Helper for creating a X86ISD::SETCC node.
Definition at line 22021 of file X86ISelLowering.cpp.
References Cond, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), and llvm::X86ISD::SETCC.
|
static |
Given the output values from getHalfShuffleMask(), create a half width shuffle of extracted vectors followed by an insert back to full width.
Definition at line 15284 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::isSimple(), llvm::Offset, and std::swap().
Referenced by lowerShuffleWithUndefHalf(), and narrowShuffle().
|
static |
Returns the scalar element that will make up the i'th element of the result of the vector shuffle.
Definition at line 6397 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::Depth, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), getShuffleScalarElt(), llvm::EVT::getSimpleVT(), getTargetShuffleMask(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::MVT::isInteger(), isTargetShuffle(), llvm::EVT::isVector(), llvm::SelectionDAG::MaxRecursionDepth, llvm::ISD::SCALAR_TO_VECTOR, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineToConsecutiveLoads(), and getShuffleScalarElt().
|
static |
Return a vector_shuffle of the specified vector of zero or undef vector.
This produces a shuffle where the low element of V2 is swizzled into the zero/undef vector, landing at element Idx. This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
Definition at line 4684 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), and Idx.
Referenced by LowerUINT_TO_FP_i32().
Definition at line 3893 of file X86ISelLowering.cpp.
References llvm::ISD::EXTRACT_SUBVECTOR, LHS, and RHS.
Referenced by combineHorizOpWithShuffle(), combinePTESTCC(), and combineSetCCMOVMSK().
Definition at line 35260 of file X86ISelLowering.cpp.
|
static |
Definition at line 4732 of file X86ISelLowering.cpp.
References llvm::any_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::assign(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::CallingConv::C, llvm::SmallVectorBase< Size_T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::extractBits(), llvm::Constant::getAggregateElement(), llvm::APInt::getAllOnes(), llvm::APInt::getBitWidth(), getBitWidth(), llvm::APInt::getBoolValue(), llvm::Type::getPrimitiveSizeInBits(), llvm::EVT::getScalarSizeInBits(), llvm::Type::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::APInt::getSplat(), getTargetConstantBitsFromNode(), getTargetConstantFromBasePtr(), getTargetConstantFromNode(), llvm::Value::getType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::APInt::insertBits(), llvm::APInt::isAllOnes(), isAnyInRange(), llvm::Type::isDoubleTy(), llvm::Type::isFloatTy(), llvm::Type::isHalfTy(), llvm::Type::isIntegerTy(), llvm::Type::isVectorTy(), llvm::peekThroughBitcasts(), Ptr, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SCALAR_TO_VECTOR, llvm::APInt::setBit(), llvm::APInt::setBits(), llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::X86ISD::VBROADCAST_LOAD, and llvm::X86ISD::VZEXT_MOVL.
Referenced by canonicalizeBitSelect(), combineAnd(), combineAndnp(), combineBitcast(), combineConcatVectorOps(), combineConstantPoolLoads(), combineExtractVectorElt(), combineMOVMSK(), combineOr(), combineOrXorWithSETCC(), combineVectorCompare(), combineVectorPack(), combineVectorShiftImm(), combineVectorShiftVar(), combineVPMADD(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), llvm::X86TargetLowering::computeKnownBitsForTargetNode(), convertShiftLeftToScale(), createShuffleMaskFromVSELECT(), getFauxShuffleMask(), getTargetConstantBitsFromNode(), getTargetShuffleAndZeroables(), getTargetShuffleMaskIndices(), llvm::X86::isConstantSplat(), isFNEG(), IsNOT(), LowerVSETCC(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
Definition at line 4707 of file X86ISelLowering.cpp.
References llvm::ConstantPoolSDNode::getConstVal(), llvm::ConstantPoolSDNode::getOffset(), getTargetConstantPoolFromBasePtr(), llvm::ConstantPoolSDNode::isMachineConstantPoolEntry(), and Ptr.
Referenced by combineConstantPoolLoads(), getTargetConstantBitsFromNode(), and getTargetConstantFromNode().
|
static |
Definition at line 4714 of file X86ISelLowering.cpp.
References getTargetConstantFromBasePtr(), and llvm::ISD::isNormalLoad().
Referenced by canonicalizeShuffleWithOp(), combineAnd(), combineConcatVectorOps(), combineMul(), getTargetConstantBitsFromNode(), llvm::X86TargetLowering::getTargetConstantFromLoad(), getTargetConstantFromNode(), lowerShuffleAsElementInsertion(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetShuffle().
Definition at line 4720 of file X86ISelLowering.cpp.
References getTargetConstantFromNode(), and llvm::peekThroughBitcasts().
|
static |
Definition at line 4699 of file X86ISelLowering.cpp.
References Ptr, llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.
Referenced by getTargetConstantFromBasePtr().
|
static |
Decode a target shuffle mask and inputs and see if any values are known to be undef or zero from their inputs.
Returns true if the target shuffle mask was decoded. FIXME: Merge this with computeZeroableShuffleElements?
Definition at line 5631 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getSizeInBits(), getTargetConstantBitsFromNode(), getTargetShuffleMask(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::APInt::getZero(), Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::isFloatingPoint(), isTargetShuffle(), llvm::SDValue::isUndef(), isUndefOrZero(), llvm::X86::isZeroNode(), N, llvm::peekThroughBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::APInt::setBit(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineTargetShuffle(), and getTargetShuffleInputs().
|
static |
Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs and then sets the SM_SentinelUndef and SM_SentinelZero values.
Returns true if the target shuffle mask was decoded.
Definition at line 6321 of file X86ISelLowering.cpp.
References llvm::Depth, getFauxShuffleMask(), getTargetShuffleAndZeroables(), llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::SelectionDAG::MaxRecursionDepth, resolveTargetShuffleFromZeroables(), and resolveZeroablesFromTargetShuffle().
|
static |
Definition at line 6347 of file X86ISelLowering.cpp.
References llvm::Depth, and getTargetShuffleInputs().
Referenced by combineEXTRACT_SUBVECTOR(), combineExtractWithShuffle(), combineHorizOpWithShuffle(), combineSetCCMOVMSK(), combineTargetShuffle(), combineX86ShufflesRecursively(), getFauxShuffleMask(), getTargetShuffleInputs(), isHorizontalBinOp(), llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(), and llvm::X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode().
|
static |
Definition at line 6357 of file X86ISelLowering.cpp.
References llvm::Depth, llvm::APInt::getAllOnes(), getTargetShuffleInputs(), llvm::EVT::isSimple(), and llvm::EVT::isVector().
|
static |
Definition at line 5533 of file X86ISelLowering.cpp.
References getTargetShuffleMask(), and N.
|
static |
Calculates the shuffle mask corresponding to the target-specific opcode.
If the mask could be calculated, returns it in Mask
, returns the shuffle operands in Ops
, and returns true. Sets IsUnary
to true if only one source is used. Note that this will set IsUnary for shuffles which use a single input multiple times, and in those cases it will adjust the mask to only have indices within that single input. It is an error to call this with non-empty Mask/Ops vectors.
Definition at line 5226 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BLENDI, llvm::DecodeBLENDMask(), llvm::DecodeEXTRQIMask(), llvm::DecodeINSERTPSMask(), llvm::DecodeINSERTQIMask(), llvm::DecodeMOVDDUPMask(), llvm::DecodeMOVHLPSMask(), llvm::DecodeMOVLHPSMask(), llvm::DecodeMOVSHDUPMask(), llvm::DecodeMOVSLDUPMask(), llvm::DecodePALIGNRMask(), llvm::DecodePSHUFBMask(), llvm::DecodePSHUFHWMask(), llvm::DecodePSHUFLWMask(), llvm::DecodePSHUFMask(), llvm::DecodePSLLDQMask(), llvm::DecodePSRLDQMask(), llvm::DecodeScalarMoveMask(), llvm::DecodeSHUFPMask(), llvm::DecodeUNPCKHMask(), llvm::DecodeUNPCKLMask(), llvm::DecodeVALIGNMask(), llvm::DecodeVectorBroadcast(), llvm::DecodeVPERM2X128Mask(), llvm::DecodeVPERMIL2PMask(), llvm::DecodeVPERMILPMask(), llvm::DecodeVPERMMask(), llvm::DecodeVPERMV3Mask(), llvm::DecodeVPERMVMask(), llvm::DecodeVPPERMMask(), llvm::decodeVSHUF64x2FamilyMask(), llvm::DecodeZeroMoveLowMask(), llvm::SmallVectorBase< Size_T >::empty(), llvm::X86ISD::EXTRQI, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), getTargetShuffleMaskIndices(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::INSERTPS, llvm::X86ISD::INSERTQI, isAnyZero(), isTargetShuffle(), llvm_unreachable, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, N, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VALIGN, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VZEXT_MOVL.
Referenced by canonicalizeShuffleWithOp(), combineBlendOfPermutes(), combineConcatVectorOps(), combineSelect(), combineTargetShuffle(), llvm::X86TargetLowering::computeKnownBitsForTargetNode(), llvm::X86TargetLowering::ComputeNumSignBitsForTargetNode(), getPSHUFShuffleMask(), getShuffleScalarElt(), getTargetShuffleAndZeroables(), getTargetShuffleMask(), and llvm::X86TargetLowering::isGuaranteedNotToBeUndefOrPoisonForTargetNode().
|
static |
Definition at line 5090 of file X86ISelLowering.cpp.
References getTargetConstantBitsFromNode(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by getTargetShuffleMask().
|
static |
Handle vector element shifts where the shift amount is a constant.
Takes immediate version of shift as input.
Definition at line 25181 of file X86ISelLowering.cpp.
References assert(), llvm::CallingConv::C, llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorElementType(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm_unreachable, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.
Referenced by combineAndMaskToShift(), combineMOVMSK(), combineVectorShiftVar(), LowerFunnelShift(), LowerMUL(), LowerMULH(), LowerMULO(), LowerRotate(), LowerShift(), LowerShiftByScalarImmediate(), and LowerShiftByScalarVariable().
|
static |
Handle vector element shifts by a splat shift amount.
Definition at line 25233 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BUILD_VECTOR, extract128BitVector(), llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasSSE41(), llvm::EVT::is128BitVector(), llvm::EVT::isSimple(), llvm::MVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, llvm::X86ISD::VZEXT_MOVL, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.
Referenced by LowerFunnelShift(), LowerRotate(), and LowerShiftByScalarVariable().
Definition at line 25161 of file X86ISelLowering.cpp.
References llvm_unreachable, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::X86ISD::VSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRA, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRL, and llvm::X86ISD::VSRLI.
Referenced by combineVectorShiftVar(), getTargetVShiftNode(), LowerShift(), LowerShiftByScalarImmediate(), and LowerShiftByScalarVariable().
|
static |
Definition at line 18663 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::X86AS::FS, llvm::PointerType::get(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::MachineFunction::getFrameInfo(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86AS::GS, llvm::Offset, Ptr, llvm::MachineFrameInfo::setAdjustsStack(), llvm::MachineFrameInfo::setHasCalls(), llvm::X86ISD::TLSADDR, llvm::X86ISD::TLSBASEADDR, llvm::X86ISD::TLSDESC, llvm::SDNode::use_begin(), llvm::SDNode::use_end(), and llvm::TargetMachine::useTLSDESC().
Referenced by LowerToTLSGeneralDynamicModel32(), LowerToTLSGeneralDynamicModel64(), LowerToTLSGeneralDynamicModelX32(), and LowerToTLSLocalDynamicModel().
For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index.
Modifies ExtractedFromVec
to the real vector and returns the real index.
Definition at line 7516 of file X86ISelLowering.cpp.
References assert(), llvm::SDNode::getAsZExtVal(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), Idx, and isUndefOrInRange().
Referenced by buildFromShuffleMostly().
|
static |
Returns a vector_shuffle node for an unpackh operation.
Definition at line 4607 of file X86ISelLowering.cpp.
References llvm::createUnpackShuffleMask(), and getVectorShuffle().
Referenced by combineArithReduction(), convertShiftLeftToScale(), LowerAVXExtend(), LowerFunnelShift(), LowerHorizontalByteSum(), LowerMUL(), LowerRotate(), LowerShift(), and LowervXi8MulWithUNPCK().
|
static |
Returns a vector_shuffle node for an unpackl operation.
Definition at line 4599 of file X86ISelLowering.cpp.
References llvm::createUnpackShuffleMask(), and getVectorShuffle().
Referenced by combineArithReduction(), convertShiftLeftToScale(), LowerFunnelShift(), LowerHorizontalByteSum(), LowerMUL(), LowerRotate(), LowerShift(), LowerUINT_TO_FP_i64(), and LowervXi8MulWithUNPCK().
Get a 4-lane 8-bit shuffle immediate for a mask.
This helper function produces an 8-bit shuffle immediate corresponding to the ubiquitous shuffle encoding scheme used in x86 instructions for shuffling 4 lanes. It can be used with most of the PSHUF instructions for example.
NB: We rely heavily on "undef" masks preserving the input lane.
Definition at line 9774 of file X86ISelLowering.cpp.
References llvm::all_of(), assert(), and llvm::find_if().
Referenced by combineVectorShiftImm(), getV4X86ShuffleImm8ForMask(), matchBinaryPermuteShuffle(), and matchUnaryPermuteShuffle().
|
static |
Definition at line 9798 of file X86ISelLowering.cpp.
References DL, llvm::SelectionDAG::getTargetConstant(), and getV4X86ShuffleImm().
Referenced by canonicalizeShuffleMaskWithHorizOp(), combineConcatVectorOps(), combineRedundantDWordShuffle(), combineTargetShuffle(), combineVectorHADDSUB(), combineVectorShiftImm(), combineX86ShuffleChain(), LowerShift(), lowerShuffleAsSpecificZeroOrAnyExtend(), lowerShuffleWithSHUFPS(), lowerV16F32Shuffle(), lowerV16I32Shuffle(), lowerV2I64Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV4X128Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I16GeneralSingleInputShuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().
|
static |
Return (and Op
, Mask
) for compare instructions or (vselect Mask
, Op
, PreservedSrc
) for others along with the necessary casting or extending for Mask
when lowering masking intrinsics.
Definition at line 25364 of file X86ISelLowering.cpp.
References getMaskNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::isAllOnesConstant(), llvm::SDValue::isUndef(), and llvm::ISD::VSELECT.
Referenced by lowerShuffleAsBlend().
|
static |
Definition at line 4578 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getBuildVector(), llvm::SDValue::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::ISD::isBuildVectorOfConstantSDNodes(), and llvm::SDValue::isUndef().
Referenced by getUnpackh(), and getUnpackl().
|
static |
Return a vector logical shift node.
Definition at line 6762 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::EVT::is128BitVector(), llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.
|
static |
Definition at line 23956 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADD, assert(), Cond, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_O, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::isOneConstant(), LHS, llvm_unreachable, RHS, llvm::ISD::SADDO, llvm::X86ISD::SMUL, llvm::ISD::SMULO, llvm::ISD::SSUBO, llvm::X86ISD::SUB, llvm::ISD::UADDO, llvm::X86ISD::UMUL, llvm::ISD::UMULO, and llvm::ISD::USUBO.
Referenced by LowerXALUO().
|
static |
Returns a vector of specified type with all zero elements.
Definition at line 3864 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isFloatingPoint(), and llvm::TargetLoweringBase::isTypeLegal().
|
static |
Definition at line 49505 of file X86ISelLowering.cpp.
Referenced by combineAndLoadToBZHI().
Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
Definition at line 2924 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86::COND_NP, and llvm::X86::COND_P.
Referenced by combineCMov().
Return true if a shuffle mask chooses elements identically in its top and bottom halves.
For example, any splat mask has the same top and bottom halves. If an element is undefined in only one half of the mask, the halves are not considered identical.
Definition at line 9756 of file X86ISelLowering.cpp.
References assert().
Referenced by LowerAVXExtend().
return true if Op
has a use that doesn't just read flags.
Definition at line 22563 of file X86ISelLowering.cpp.
References llvm::ISD::BRCOND, llvm::Value::hasOneUse(), llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::TRUNCATE, and llvm::Value::use_begin().
|
static |
Given a buildvector constant, return a new vector constant with each element incremented or decremented.
If incrementing or decrementing would result in unsigned overflow or underflow or this is not a simple vector constant, return an empty value.
Definition at line 23157 of file X86ISelLowering.cpp.
References DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::APInt::isMaxSignedValue(), llvm::APInt::isMaxValue(), llvm::APInt::isMinSignedValue(), llvm::APInt::isZero(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by combineSetCC(), LowerVSETCC(), and LowerVSETCCWithSUBUS().
|
static |
Generate a DAG to put 128-bits into a vector > 128 bits.
This sets things up to match to an AVX VINSERTF128/VINSERTI128 or AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a simple superregister reference. Idx is an index in the 128 bits we want. It need not be aligned to a 128-bit boundary. That makes lowering INSERT_VECTOR_ELT operations easier.
Definition at line 3995 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getValueType(), insertSubVector(), and llvm::EVT::is128BitVector().
Referenced by LowerSCALAR_TO_VECTOR(), and lowerV4X128Shuffle().
|
static |
Insert i1-subvector to i1-vector.
Definition at line 4325 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::flipAllBits(), llvm::APInt::getBitsSet(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), High, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), llvm::X86ISD::KSHIFTL, llvm::X86ISD::KSHIFTR, llvm::Low, llvm::SDNode::ops(), llvm::ISD::OR, and widenMaskVectorType().
Referenced by LowerINSERT_SUBVECTOR().
|
static |
Insert one bit to mask vector, like v16i1 or v8i1.
AVX-512 feature.
Definition at line 18177 of file X86ISelLowering.cpp.
References llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SIGN_EXTEND, and llvm::ISD::TRUNCATE.
|
static |
Definition at line 3965 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_SUBVECTOR, llvm::isPowerOf2_32(), and llvm::SDValue::isUndef().
Referenced by combineSelect(), combineX86ShuffleChain(), concatSubVectors(), getHopForBuildVector(), insert128BitVector(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
Test whether there are elements crossing 128-bit lanes in this shuffle mask.
Definition at line 9422 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), and isLaneCrossingShuffleMask().
Referenced by combineX86ShuffleChain(), lowerShuffleAsBlendOfPSHUFBs(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), and matchUnaryPermuteShuffle().
Definition at line 9502 of file X86ISelLowering.cpp.
References isRepeatedShuffleMask().
|
static |
Test whether a shuffle mask is equivalent within each 128-bit lane.
Definition at line 9496 of file X86ISelLowering.cpp.
References isRepeatedShuffleMask().
Referenced by combineBlendOfPermutes(), LowerShift(), lowerShuffleAsBlend(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV32I16Shuffle(), lowerV4I64Shuffle(), lowerV8F32Shuffle(), lowerV8I32Shuffle(), lowerV8I64Shuffle(), matchShuffleAsByteRotate(), and matchUnaryPermuteShuffle().
|
static |
Definition at line 9731 of file X86ISelLowering.cpp.
References llvm::ShuffleVectorSDNode::commuteMask(), llvm::createUnpackShuffleMask(), llvm::MVT::getIntegerVT(), llvm::MVT::getVectorVT(), and isTargetShuffleEquivalent().
Referenced by lowerShuffleOfExtractsAsVperm(), and lowerShuffleWithUndefHalf().
|
static |
Test whether a shuffle mask is equivalent within each 256-bit lane.
Definition at line 9509 of file X86ISelLowering.cpp.
References isRepeatedShuffleMask().
Referenced by lowerV8F64Shuffle(), lowerV8I64Shuffle(), and matchUnaryPermuteShuffle().
|
static |
Returns true iff BV
builds a vector with the result equivalent to the result of ADDSUB/SUBADD operation.
If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1 (SUBADD = Opnd0 -+ Opnd1) operation are written to the parameters Opnd0
and Opnd1
.
Definition at line 7911 of file X86ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE3(), llvm::MVT::isFloatingPoint(), llvm::SDValue::isUndef(), std::swap(), and llvm::ISD::UNDEF.
Referenced by combineShuffleToAddSubOrFMAddSub(), and lowerToAddSubOrFMAddSub().
|
static |
Returns true iff the shuffle node N
can be replaced with ADDSUB(SUBADD) operation.
If true is returned then the operands of ADDSUB(SUBADD) operation are written to the parameters Opnd0
and Opnd1
.
We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes so it is easier to generically match. We also insert dummy vector shuffle nodes for the operands which explicitly discard the lanes which are unused by this operation to try to flow through the rest of the combiner the fact that they're unused.
Definition at line 41562 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::hasOneUse(), llvm::X86Subtarget::hasSSE3(), isAddSubOrSubAddMask(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, RHS, and llvm::ISD::VECTOR_SHUFFLE.
Checks if the shuffle mask takes subsequent elements alternately from two vectors.
For example <0, 5, 2, 7> or <8, 1, 10, 3, 12, 5, 14, 7> are both correct.
Definition at line 41525 of file X86ISelLowering.cpp.
References Size.
Referenced by combineShuffleToFMAddSub(), and isAddSubOrSubAdd().
Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR.
Definition at line 24791 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::OR, and llvm::X86ISD::SETCC.
Return true if the value of any element in Mask falls within the specified range (L, H].
Definition at line 3577 of file X86ISelLowering.cpp.
References llvm::any_of(), llvm::Hi, isInRange(), and llvm::Low.
Referenced by getTargetConstantBitsFromNode().
Return true if the value of any element in Mask is the zero sentinel value.
Definition at line 3582 of file X86ISelLowering.cpp.
References llvm::any_of(), and llvm::SM_SentinelZero.
Referenced by combineHorizOpWithShuffle(), combineX86ShuffleChain(), getTargetShuffleMask(), isHorizontalBinOp(), matchBinaryPermuteShuffle(), matchShuffleAsByteRotate(), and matchUnaryPermuteShuffle().
|
static |
Return true if every element in Mask, is an in-place blend/select mask or is undef.
Definition at line 3614 of file X86ISelLowering.cpp.
References llvm::enumerate(), I, and isUndefOrEqual().
Referenced by combineBlendOfPermutes().
Definition at line 11123 of file X86ISelLowering.cpp.
References isUndefOrEqual().
Referenced by isNoopOrBroadcastShuffleMask(), lower1BitShuffle(), and lowerShuffleAsDecomposedShuffleMerge().
|
static |
Definition at line 34871 of file X86ISelLowering.cpp.
References MI.
Return true if every element of a single input is referenced by the shuffle mask.
i.e. it just permutes them all.
Definition at line 3654 of file X86ISelLowering.cpp.
References llvm::APInt::getZero(), llvm::APInt::isAllOnes(), isInRange(), and llvm::APInt::setBit().
Referenced by combineSetCCMOVMSK().
|
static |
Definition at line 34468 of file X86ISelLowering.cpp.
References llvm::MachineBasicBlock::end(), llvm::make_range(), and llvm::MachineBasicBlock::successors().
Referenced by checkAndUpdateEFLAGSKill(), and emitXBegin().
|
static |
Checks whether the vector elements referenced by two shuffle masks are equivalent.
Definition at line 9562 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::SDValue::getNumOperands(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::X86ISD::HSUB, Idx, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VBROADCAST_LOAD.
Referenced by isShuffleEquivalent(), isTargetShuffleEquivalent(), and matchShuffleAsBlend().
|
static |
Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBADD(Opnd0
, Opnd1
) into FMADDSUB/FMSUBADD(x, y, Opnd1
).
If (and only if) true is returned, the operands of FMADDSUB/FMSUBADD are written to parameters Opnd0
, Opnd1
, Opnd2
.
Prior to calling this function it should be known that there is some SDNode that potentially can be replaced with an X86ISD::ADDSUB operation using Opnd0
and Opnd1
as operands. Also, this method is called before replacement of such SDNode with ADDSUB operation. Thus the number of Opnd0
uses is expected to be equal to 2. For example, this function may be called for the following IR: AB = fmul fast <2 x double> A, B Sub = fsub fast <2 x double> AB, C Add = fadd fast <2 x double> AB, C Addsub = shufflevector <2 x double> Sub, <2 x double> Add, <2 x i32> <i32 0, i32 3> There is a def for Addsub here, which potentially can be replaced by X86ISD::ADDSUB operation: Addsub = X86ISD::ADDSUB AB, C and such ADDSUB can further be replaced with FMADDSUB: Addsub = FMADDSUB A, B, C.
The main reason why this method is called before the replacement of the recognized ADDSUB idiom with ADDSUB operation is that such replacement is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit FMADDSUB is.
Definition at line 8038 of file X86ISelLowering.cpp.
References llvm::FPOpFusion::Fast, llvm::ISD::FMUL, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTarget(), llvm::X86Subtarget::hasAnyFMA(), llvm::SDNode::hasNUsesOfValue(), llvm::TargetMachine::Options, and Options.
Referenced by combineShuffleToAddSubOrFMAddSub(), and lowerToAddSubOrFMAddSub().
|
static |
Returns the negated value if the node N
flips sign of FP value.
FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000) or FSUB(0, x) AVX512F does not have FXOR, so FNEG is lowered as (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))). In this case we go though all bitcasts. This also recognizes splat of a negated value and returns the splat of that value.
Definition at line 52727 of file X86ISelLowering.cpp.
References llvm::Depth, llvm::ISD::FNEG, llvm::ISD::FSUB, llvm::X86ISD::FXOR, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::ISD::INSERT_VECTOR_ELT, isFNEG(), llvm::SDValue::isUndef(), llvm::SelectionDAG::MaxRecursionDepth, N, llvm::peekThroughBitcasts(), llvm::SmallVectorBase< Size_T >::size(), std::swap(), llvm::ISD::VECTOR_SHUFFLE, and llvm::ISD::XOR.
Referenced by combineFneg(), llvm::X86TargetLowering::getNegatedExpression(), and isFNEG().
Definition at line 7248 of file X86ISelLowering.cpp.
References llvm::ISD::BITCAST, isFoldableUseOfShuffle(), isTargetShuffle(), N, llvm::X86ISD::VPDPBUSD, llvm::X86ISD::VPERMV, and llvm::X86ISD::VPERMV3.
Referenced by isFoldableUseOfShuffle(), and lowerBuildVectorAsBroadcast().
|
static |
Check if RC
is a vector register class.
I.e., FR* / VR* or one of their variant.
Definition at line 58849 of file X86ISelLowering.cpp.
References llvm::TargetRegisterClass::hasSuperClassEq().
Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().
|
static |
Definition at line 4136 of file X86ISelLowering.cpp.
References collectConcatOps(), and N.
Referenced by LowerStore(), matchTruncateWithPACK(), narrowExtractedVectorSelect(), and narrowVectorSelect().
|
static |
Check if RC
is a general purpose register class.
I.e., GR* or one of their variant.
Definition at line 58839 of file X86ISelLowering.cpp.
References llvm::TargetRegisterClass::hasSuperClassEq().
Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().
|
static |
Definition at line 8106 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::DELETED_NODE, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::X86ISD::HSUB, llvm::MVT::is256BitVector(), llvm::SDValue::isUndef(), and llvm::ISD::SUB.
Referenced by LowerToHorizontalOp().
|
static |
Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS.
A horizontal operation performs the binary operation on successive elements of its first operand, then on successive elements of its second operand, returning the resulting values in a vector. For example, if A = < float a0, float a1, float a2, float a3 > and B = < float b0, float b1, float b2, float b3 > then the result of doing a horizontal operation on A and B is A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >. In short, LHS and RHS are inspected to see if LHS op RHS is of the form A horizontal-op B, for some already available A and B, and if so then LHS is set to A, RHS to B, and the routine returns 'true'.
Definition at line 51946 of file X86ISelLowering.cpp.
References A, llvm::all_of(), llvm::any_of(), llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::assign(), B, llvm::sampleprof::Base, llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::CallingConv::C, llvm::SmallVectorImpl< T >::clear(), llvm::ShuffleVectorSDNode::commuteMask(), D, llvm::SmallVectorBase< Size_T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), getTargetShuffleInputs(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), isAnyZero(), llvm::MVT::isFloatingPoint(), isMultiLaneShuffleMask(), llvm::isNullConstant(), isSequentialOrUndefInRange(), isUndefOrInRange(), LHS, llvm::peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), resolveTargetShuffleInputsAndMask(), RHS, scaleShuffleElements(), shouldUseHorizontalOp(), llvm::SmallVectorBase< Size_T >::size(), llvm::ArrayRef< T >::slice(), llvm::SM_SentinelUndef, llvm::SelectionDAG::SplitVector(), std::swap(), and llvm::SDNode::uses().
Referenced by combineToHorizontalAddSub().
|
static |
This is a helper function of LowerToHorizontalOp().
This function checks that the build_vector N
in input implements a 128-bit partial horizontal operation on a 256-bit vector, but that operation may not match the layout of an x86 256-bit horizontal instruction. In other words, if this returns true, then some extraction/insertion will be required to produce a valid horizontal instruction.
Parameter Opcode
defines the kind of horizontal operation to match. For example, if Opcode
is equal to ISD::ADD, then this function checks if N
implements a horizontal arithmetic add; if instead Opcode
is equal to ISD::SUB, then this function checks if this is a horizontal arithmetic sub.
This function only analyzes elements of N
whose indices are in range [BaseIdx, LastIdx).
TODO: This function was originally used to match both real and fake partial horizontal operations, but the index-matching logic is incorrect for that. See the corrected implementation in isHopBuildVector(). Can we reduce this code because it is only used for partial h-op matching now?
Definition at line 7756 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::is256BitVector(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), and N.
Referenced by LowerToHorizontalOp().
|
static |
Definition at line 7723 of file X86ISelLowering.cpp.
References llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::X86ISD::HADD, llvm::X86ISD::HSUB, llvm::X86ISD::PACKSS, and llvm::X86ISD::PACKUS.
Referenced by combineHorizOpWithShuffle().
|
static |
Return true if Val falls within the specified range (L, H].
Definition at line 3571 of file X86ISelLowering.cpp.
References llvm::Hi, and llvm::Low.
Referenced by canonicalizeShuffleMaskWithHorizOp(), combineX86ShuffleChain(), combineX86ShufflesRecursively(), getFauxShuffleMask(), isAnyInRange(), isCompletePermute(), isTargetShuffleEquivalent(), isUndefOrInRange(), and isUndefOrZeroOrInRange().
|
static |
Test whether there are elements crossing LaneSizeInBits lanes in this shuffle mask.
X86 divides up its shuffles into in-lane and cross-lane shuffle operations and we routinely test for these.
Definition at line 9406 of file X86ISelLowering.cpp.
References assert(), and Size.
Referenced by combineBlendOfPermutes(), and is128BitLaneCrossingShuffleMask().
|
static |
Definition at line 19318 of file X86ISelLowering.cpp.
References llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE2(), and llvm::X86Subtarget::useAVX512Regs().
Definition at line 2688 of file X86ISelLowering.cpp.
References llvm::X86ISD::ANDNP, and llvm::ISD::isBitwiseLogicOp().
Referenced by canonicalizeShuffleWithOp().
|
static |
Test whether elements in each LaneSizeInBits lane in this shuffle mask come from multiple lanes - this is different to isLaneCrossingShuffleMask to better support 'repeated mask + lane permute' style shuffles.
Definition at line 9429 of file X86ISelLowering.cpp.
References assert().
Referenced by isHorizontalBinOp().
|
static |
Definition at line 9810 of file X86ISelLowering.cpp.
References assert().
Referenced by lowerShuffleToEXPAND().
Definition at line 11127 of file X86ISelLowering.cpp.
References isBroadcastShuffleMask(), and isNoopShuffleMask().
Referenced by lowerShuffleAsDecomposedShuffleMerge().
Tiny helper function to identify a no-op mask.
This is a somewhat boring predicate function. It checks whether the mask array input, which is assumed to be a single-input shuffle mask of the kind used by the X86 shuffle instructions (not a fully general ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an in-place shuffle are 'no-op's.
Definition at line 9392 of file X86ISelLowering.cpp.
References assert(), and Size.
Referenced by combineTargetShuffle(), isNoopOrBroadcastShuffleMask(), lowerShuffleAsByteRotate(), lowerShuffleAsByteShiftMask(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsPermuteAndUnpack(), lowerV8I16GeneralSingleInputShuffle(), and matchShuffleAsBitRotate().
|
static |
Definition at line 5112 of file X86ISelLowering.cpp.
References collectConcatOps(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), getTargetConstantBitsFromNode(), llvm::isAllOnesConstant(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), IsNOT(), llvm::isNullConstant(), llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), and llvm::ISD::XOR.
Referenced by combineAndNotIntoANDNP(), combineAndnp(), combineAndShuffleNot(), combineMOVMSK(), combinePTESTCC(), combineSelect(), combineVectorPack(), and IsNOT().
Definition at line 53237 of file X86ISelLowering.cpp.
References llvm::ISD::isBuildVectorAllZeros(), and llvm::isNullFPConstant().
Referenced by combineFAndn(), combineFOr(), combineSetCC(), and getNullFPConstForNullVal().
Recursive helper for combineVectorSizedSetCCEquality() to see if we have a recognizable memcmp expansion.
Definition at line 22029 of file X86ISelLowering.cpp.
References isOrXorXorTree(), llvm::ISD::OR, X, and llvm::ISD::XOR.
Referenced by combineVectorSizedSetCCEquality(), and isOrXorXorTree().
Definition at line 22584 of file X86ISelLowering.cpp.
References llvm::ISD::CopyToReg, llvm::ISD::SETCC, and llvm::ISD::STORE.
Referenced by EmitTest().
|
static |
Test whether a shuffle mask is equivalent within each sub-lane.
This checks a shuffle mask to see if it is performing the same lane-relative shuffle in each sub-lane. This trivially implies that it is also not lane-crossing. It may however involve a blend from the same lane of a second vector.
The specific repeated shuffle mask is populated in RepeatedMask
, as it is non-trivial to compute in the face of undef lanes. The representation is suitable for use with existing 128-bit shuffles as entries from the second vector have been remapped to [LaneSize, 2*LaneSize).
Definition at line 9466 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::MVT::getScalarSizeInBits(), Size, and llvm::SM_SentinelUndef.
Referenced by is128BitLaneRepeatedShuffleMask(), and is256BitLaneRepeatedShuffleMask().
|
static |
Test whether a target shuffle mask is equivalent within each sub-lane.
Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.
Definition at line 9553 of file X86ISelLowering.cpp.
References llvm::MVT::getScalarSizeInBits(), and isRepeatedTargetShuffleMask().
|
static |
Test whether a target shuffle mask is equivalent within each sub-lane.
Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.
Definition at line 9516 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::assign(), isUndefOrZero(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by canonicalizeShuffleMaskWithHorizOp(), isRepeatedTargetShuffleMask(), and matchBinaryPermuteShuffle().
|
static |
Return true if every element in Mask, beginning from position Pos and ending in Pos + Size, falls within the specified sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.
Definition at line 3625 of file X86ISelLowering.cpp.
References isUndefOrEqual(), llvm::Low, and Size.
Referenced by combineX86ShuffleChain(), isHorizontalBinOp(), lowerShuffleAsByteShiftMask(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsVALIGN(), lowerShuffleAsVTRUNC(), lowerShuffleAsVTRUNCAndUnpack(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithUndefHalf(), lowerShuffleWithVPMOV(), lowerV8I16GeneralSingleInputShuffle(), match1BitShuffleAsKSHIFT(), matchShuffleAsINSERTQ(), matchShuffleAsShift(), matchShuffleAsVTRUNC(), matchUnaryPermuteShuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
|
static |
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero.
Definition at line 3636 of file X86ISelLowering.cpp.
References isUndefOrZero(), llvm::Low, and Size.
Referenced by combineX86ShuffleChain(), and matchShuffleWithUNPCK().
|
static |
Checks whether a shuffle mask is equivalent to an explicit list of arguments.
This is a fast way to test a shuffle mask against a fixed pattern:
if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }
It returns true if the mask is exactly as wide as the argument list, and each element of the mask is either -1 (signifying undef) or the value given in the argument.
Definition at line 9622 of file X86ISelLowering.cpp.
References assert(), IsElementEquivalent(), llvm::ArrayRef< T >::size(), and Size.
Referenced by lowerShuffleWithUNPCK(), lowerShuffleWithUNPCK256(), lowerV16F32Shuffle(), lowerV2F64Shuffle(), lowerV2X128Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4X128Shuffle(), lowerV8F32Shuffle(), and lowerV8F64Shuffle().
Helper to test for a load that can be folded with x86 shuffles.
This is particularly important because the set of instructions varies significantly based on whether the operand is a load or not.
Definition at line 12201 of file X86ISelLowering.cpp.
References getNode(), llvm::ISD::isNON_EXTLoad(), and llvm::peekThroughOneUseBitcasts().
Referenced by canonicalizeShuffleWithOp(), and lowerShuffleAsBroadcast().
Test whether the specified input (0 or 1) is in-place blended by the given mask.
This returns true if the elements from a particular input are already in the slot required by the given mask and require no permutation.
Definition at line 12418 of file X86ISelLowering.cpp.
References assert(), and Size.
Referenced by lowerV4F64Shuffle(), and lowerV4I64Shuffle().
Check if the Mask consists of the same element repeated multiple times.
Definition at line 11132 of file X86ISelLowering.cpp.
References llvm::SM_SentinelUndef.
Referenced by lowerShuffleAsDecomposedShuffleMerge().
Test whether this can be lowered with a single SHUFPS instruction.
This is used to disable more specialized lowerings when the shufps lowering will happen to be efficient.
Definition at line 12395 of file X86ISelLowering.cpp.
References assert().
Referenced by lowerShuffleOfExtractsAsVperm(), lowerShuffleWithUndefHalf(), lowerV16I32Shuffle(), lowerV4F32Shuffle(), lowerV4I32Shuffle(), and lowerV8I32Shuffle().
|
static |
Definition at line 12207 of file X86ISelLowering.cpp.
Referenced by combineFMinNumFMaxNum(), combineSelect(), combineShuffle(), lowerShuffleAsElementInsertion(), LowerVSETCC(), and llvm::X86TargetLowering::ReplaceNodeResults().
Definition at line 2693 of file X86ISelLowering.cpp.
References llvm::X86ISD::BLENDI, llvm::X86ISD::EXTRQI, llvm::X86ISD::INSERTPS, llvm::X86ISD::INSERTQI, llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::MOVSS, llvm::X86ISD::PALIGNR, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::X86ISD::SHUF128, llvm::X86ISD::SHUFP, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VALIGN, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VPERM2X128, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, llvm::X86ISD::VPPERM, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VZEXT_MOVL.
Referenced by canonicalizeShuffleWithOp(), combineINSERT_SUBVECTOR(), combineSelect(), combineShuffle(), combineTargetShuffle(), llvm::X86TargetLowering::computeKnownBitsForTargetNode(), llvm::X86TargetLowering::ComputeNumSignBitsForTargetNode(), getShuffleScalarElt(), getTargetShuffleAndZeroables(), getTargetShuffleMask(), and isFoldableUseOfShuffle().
|
static |
Checks whether a target shuffle mask is equivalent to an explicit pattern.
The masks must be exactly the same width.
If an element in Mask matches SM_SentinelUndef (-1) then the corresponding value in ExpectedMask is always accepted. Otherwise the indices must match.
SM_SentinelZero is accepted as a valid negative index but must match in both, or via a known bits test.
Definition at line 9654 of file X86ISelLowering.cpp.
References llvm::all_of(), assert(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), IsElementEquivalent(), isInRange(), isUndefOrZeroOrInRange(), llvm::APInt::isZero(), llvm::SelectionDAG::MaskedVectorIsZero(), llvm::APInt::setBit(), llvm::ArrayRef< T >::size(), Size, llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineX86ShuffleChain(), is128BitUnpackShuffleMask(), isUnpackWdShuffleMask(), matchBinaryShuffle(), matchShuffleWithPACK(), matchShuffleWithUNPCK(), and matchUnaryShuffle().
Definition at line 2734 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::ISD::OR, llvm::X86ISD::PSHUFB, llvm::X86ISD::VPERMIL2, llvm::X86ISD::VPERMILPV, llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, and llvm::X86ISD::VPPERM.
Referenced by combineX86ShufflesRecursively().
|
static |
Definition at line 24031 of file X86ISelLowering.cpp.
References llvm::APInt::getHighBitsSet(), llvm::SDValue::getValueSizeInBits(), llvm::SelectionDAG::MaskedValueIsZero(), and llvm::ISD::TRUNCATE.
Definition at line 40498 of file X86ISelLowering.cpp.
References llvm::ISD::CTLZ, llvm::ISD::CTPOP, and llvm::ISD::CTTZ.
Referenced by canonicalizeShuffleWithOp().
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value.
Definition at line 3553 of file X86ISelLowering.cpp.
References llvm::all_of(), and Size.
Referenced by canonicalizeShuffleMaskWithHorizOp(), isUndefLowerHalf(), isUndefUpperHalf(), lowerShuffleAsVTRUNC(), lowerShuffleWithVPMOV(), matchBinaryPermuteShuffle(), matchShuffleAsINSERTQ(), matchUnaryShuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
Return true if the mask creates a vector whose lower half is undefined.
Definition at line 3559 of file X86ISelLowering.cpp.
References isUndefInRange().
Referenced by getHalfShuffleMask(), and lowerShuffleWithUndefHalf().
Return true if every element in Mask is the undef sentinel value or equal to the specified value.
Definition at line 3532 of file X86ISelLowering.cpp.
References llvm::all_of(), and llvm::SM_SentinelUndef.
|
static |
Val is the undef sentinel value or equal to the specified value.
Definition at line 3526 of file X86ISelLowering.cpp.
References llvm::SM_SentinelUndef.
Referenced by combineBlendOfPermutes(), combineX86ShuffleChain(), isBlendOrUndef(), isBroadcastShuffleMask(), isSequentialOrUndefInRange(), lowerShuffleAsLanePermuteAndPermute(), lowerV8I16GeneralSingleInputShuffle(), matchUnaryShuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
|
static |
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value or equal to the specified value.
Definition at line 3540 of file X86ISelLowering.cpp.
References llvm::all_of(), and Size.
Referenced by matchUnaryShuffle().
Return true if every element in Mask is undef or if its value falls within the specified range (L, H].
Definition at line 3594 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::Hi, isUndefOrInRange(), and llvm::Low.
|
static |
Return true if Val is undef or if its value falls within the specified range (L, H].
Definition at line 3588 of file X86ISelLowering.cpp.
References llvm::Hi, isInRange(), llvm::Low, and llvm::SM_SentinelUndef.
Referenced by combineBlendOfPermutes(), combineTargetShuffle(), combineX86ShuffleChain(), getFauxShuffleMask(), getUnderlyingExtractedFromVec(), isHorizontalBinOp(), isUndefOrInRange(), lowerShuffleAsByteShiftMask(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsUNPCKAndPermute(), lowerV8I16GeneralSingleInputShuffle(), matchBinaryPermuteShuffle(), and matchUnaryPermuteShuffle().
|
static |
Val is either the undef or zero sentinel value.
Definition at line 3547 of file X86ISelLowering.cpp.
References llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by canonicalizeShuffleMaskWithHorizOp(), combineSelect(), combineX86ShuffleChain(), combineX86ShufflesRecursively(), getTargetShuffleAndZeroables(), isRepeatedTargetShuffleMask(), isSequentialOrUndefOrZeroInRange(), isUndefOrZeroInRange(), isUndefOrZeroOrInRange(), matchShuffleWithUNPCK(), and matchUnaryShuffle().
Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero.
Definition at line 3647 of file X86ISelLowering.cpp.
References llvm::all_of(), isUndefOrZero(), and Size.
Referenced by combineExtractWithShuffle(), combineX86ShuffleChain(), matchBinaryPermuteShuffle(), matchUnaryShuffle(), and llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode().
Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H].
Definition at line 3607 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::Hi, isUndefOrZeroOrInRange(), and llvm::Low.
|
static |
Return true if Val is undef, zero or if its value falls within the specified range (L, H].
Definition at line 3601 of file X86ISelLowering.cpp.
References llvm::Hi, isInRange(), isUndefOrZero(), and llvm::Low.
Referenced by canonicalizeShuffleMaskWithHorizOp(), isTargetShuffleEquivalent(), isUndefOrZeroOrInRange(), and matchShuffleWithSHUFPD().
Return true if the mask creates a vector whose upper half is undefined.
Definition at line 3565 of file X86ISelLowering.cpp.
References isUndefInRange().
Referenced by getHalfShuffleMask(), lowerShuffleAsSpecificZeroOrAnyExtend(), lowerShuffleWithUndefHalf(), matchShuffleAsEXTRQ(), matchShuffleAsINSERTQ(), and narrowShuffle().
|
static |
Definition at line 9715 of file X86ISelLowering.cpp.
References llvm::createUnpackShuffleMask(), and isTargetShuffleEquivalent().
Referenced by lowerV8F32Shuffle(), and lowerV8I32Shuffle().
|
static |
Definition at line 4115 of file X86ISelLowering.cpp.
References llvm::any_of(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), collectConcatOps(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by LowerTruncateVecPack(), and LowerTruncateVecPackWithSignBits().
|
static |
Check if RC
is a mask register class.
I.e., VK* or one of their variant.
Definition at line 58860 of file X86ISelLowering.cpp.
References llvm::TargetRegisterClass::hasSuperClassEq().
Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().
Return true if the condition is an signed comparison operation.
Definition at line 2801 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, and llvm_unreachable.
Referenced by EmitCmp().
Return true if opcode is a X86 logical comparison.
Definition at line 24017 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::X86ISD::ADD, llvm::X86ISD::AND, llvm::X86ISD::CMP, llvm::X86ISD::COMI, llvm::X86ISD::FCMP, llvm::X86ISD::OR, llvm::X86ISD::SBB, llvm::X86ISD::SMUL, llvm::X86ISD::SUB, llvm::X86ISD::UCOMI, llvm::X86ISD::UMUL, and llvm::X86ISD::XOR.
|
static |
Dispatching routine to lower various 128-bit x86 vector shuffles.
This routine breaks down the specific type of 128-bit shuffle and dispatches to the lowering routines accordingly.
Definition at line 14501 of file X86ISelLowering.cpp.
References DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getVectorShuffle(), llvm_unreachable, lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV4F32Shuffle(), lowerV4I32Shuffle(), lowerV8F16Shuffle(), lowerV8I16Shuffle(), and llvm::MVT::SimpleTy.
|
static |
Definition at line 17349 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), CC, llvm::count_if(), llvm::APInt::countl_one(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::SDNode::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, isBroadcastShuffleMask(), llvm::X86ISD::KSHIFTL, llvm::X86ISD::KSHIFTR, llvm_unreachable, lower1BitShuffleAsKSHIFTR(), match1BitShuffleAsKSHIFT(), llvm::Offset, llvm::ISD::SETCC, llvm::ISD::SETGT, llvm::ISD::SIGN_EXTEND, llvm::MVT::SimpleTy, llvm::ISD::TRUNCATE, llvm::X86Subtarget::useBWIRegs(), and widenMaskVector().
Referenced by lowerVECTOR_SHUFFLE().
|
static |
Definition at line 17274 of file X86ISelLowering.cpp.
References assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::X86ISD::KSHIFTR, llvm::SM_SentinelUndef, and widenMaskVector().
Referenced by lower1BitShuffle().
|
static |
High-level routine to lower various 256-bit x86 vector shuffles.
This routine either breaks down the specific type of a 256-bit x86 vector shuffle or splits it into two 128-bit shuffles and fuses the results back together based on the available instructions.
Definition at line 16569 of file X86ISelLowering.cpp.
References llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::isInteger(), llvm_unreachable, lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsElementInsertion(), lowerShuffleWithUndefHalf(), lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV4I64Shuffle(), lowerV8F32Shuffle(), lowerV8I32Shuffle(), llvm::MVT::SimpleTy, and splitAndLowerShuffle().
|
static |
High-level routine to lower various 512-bit x86 vector shuffles.
This routine either breaks down the specific type of a 512-bit x86 vector shuffle or splits it into two 256-bit shuffles and fuses the results back together based on the available instructions.
Definition at line 17200 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX512(), llvm_unreachable, lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsBroadcast(), lowerShuffleAsElementInsertion(), lowerShuffleWithUndefHalf(), lowerV16F32Shuffle(), lowerV16I32Shuffle(), lowerV32I16Shuffle(), lowerV64I8Shuffle(), lowerV8F64Shuffle(), lowerV8I64Shuffle(), llvm::MVT::SimpleTy, and splitAndLowerShuffle().
Referenced by lowerVECTOR_SHUFFLE().
|
static |
Definition at line 28423 of file X86ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABS, llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::X86Subtarget::hasInt256(), llvm::MVT::is256BitVector(), llvm::MVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), LHS, RHS, llvm::ISD::SIGN_EXTEND, splitVectorIntBinary(), llvm::ISD::SUB, llvm::ISD::TRUNCATE, llvm::X86Subtarget::useBWIRegs(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 28207 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BLENDV, llvm::X86ISD::CMOV, llvm::X86::COND_NS, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is256BitVector(), llvm::MVT::isInteger(), splitVectorIntUnary(), and llvm::X86ISD::SUB.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 32190 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), N, llvm::X86AS::PTR32_UPTR, llvm::report_fatal_error(), llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Definition at line 28137 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::BitWidth, llvm::CallingConv::C, llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignedMinValue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasInt256(), llvm::MVT::is256BitVector(), llvm::isConstOrConstSplat(), llvm::TargetLoweringBase::isOperationLegal(), llvm::MVT::isVector(), llvm::ISD::SADDO, llvm::ISD::SADDSAT, llvm::ISD::SETLT, llvm::ISD::SETUGT, splitVectorIntBinary(), llvm::ISD::SRA, llvm::ISD::SSUBO, llvm::ISD::SSUBSAT, llvm::ISD::SUB, llvm::ISD::UMAX, useVPTERNLOG(), llvm::ISD::USUBSAT, X, llvm::Xor, llvm::ISD::XOR, and Y.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 28120 of file X86ISelLowering.cpp.
References assert(), DL, lowerAddSubToHorizontalOp(), and splitVectorIntBinary().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 31829 of file X86ISelLowering.cpp.
References llvm::X86ISD::ADC, llvm::X86ISD::ADD, llvm::X86::COND_B, llvm::X86::COND_O, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getNode(), getSETCC(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::MERGE_VALUES, N, llvm::ISD::SADDO_CARRY, llvm::X86ISD::SBB, llvm::ISD::SSUBO_CARRY, llvm::ISD::TRUNCATE, and llvm::ISD::UADDO_CARRY.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation.
Definition at line 21725 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::BitWidth, DL, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::Value::hasOneUse(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, LHS, llvm_unreachable, RHS, shouldUseHorizontalOp(), llvm::ISD::SUB, std::swap(), and X.
Referenced by lowerAddSub().
|
static |
Definition at line 27473 of file X86ISelLowering.cpp.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Result of 'and' is compared against zero.
Change to a BT node if possible. Returns the BT node and the condition code needed to use it.
Definition at line 22972 of file X86ISelLowering.cpp.
References llvm::And, llvm::ISD::AND, assert(), llvm::BitWidth, CC, llvm::SelectionDAG::computeKnownBits(), llvm::X86::COND_AE, llvm::X86::COND_B, llvm::ISD::Constant, llvm::KnownBits::countMinLeadingZeros(), getBT(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::ConstantSDNode::getZExtValue(), llvm::isBitwiseNot(), llvm::isOneConstant(), llvm::isPowerOf2_64(), llvm::Log2_64_Ceil(), llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::SHL, llvm::SelectionDAG::shouldOptForSize(), llvm::ISD::SRL, std::swap(), and llvm::ISD::TRUNCATE.
Referenced by combineAddOrSubToADCOrSBB().
|
static |
Definition at line 24407 of file X86ISelLowering.cpp.
References assert(), DL, llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX(), LowerAVXExtend(), and LowerSIGN_EXTEND_Mask().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 6774 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFrameInfo(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::InferPtrAlign(), llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::MachineFrameInfo::isFixedObjectIndex(), llvm::ISD::isNormalLoad(), llvm::Offset, Ptr, llvm::MachineFrameInfo::setObjectAlignment(), and llvm::Align::value().
|
static |
Definition at line 31059 of file X86ISelLowering.cpp.
References emitLockedStackOp(), llvm::SelectionDAG::getNode(), llvm::X86Subtarget::hasMFence(), llvm::ISD::MEMBARRIER, llvm::X86ISD::MFENCE, llvm::SequentiallyConsistent, and llvm::SyncScope::System.
|
static |
Definition at line 31747 of file X86ISelLowering.cpp.
References llvm::ISD::ATOMIC_SWAP, llvm::SelectionDAG::CreateStackTemporary(), emitLockedStackOp(), llvm::X86ISD::FILD, llvm::X86ISD::FIST, llvm::SelectionDAG::getAtomic(), llvm::SelectionDAG::getBitcast(), llvm::MachinePointerInfo::getFixedStack(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX(), llvm::Function::hasFnAttribute(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MachineMemOperand::MOLoad, llvm::MachineMemOperand::MOStore, llvm::ISD::SCALAR_TO_VECTOR, llvm::SequentiallyConsistent, and llvm::X86ISD::VEXTRACT_STORE.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Lower atomic_load_ops into LOCK-prefixed operations.
Definition at line 31679 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_OR, llvm::ISD::ATOMIC_LOAD_SUB, llvm::ISD::ATOMIC_LOAD_XOR, DL, emitLockedStackOp(), llvm::SelectionDAG::getAtomic(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getSuccessOrdering(), llvm::MemSDNode::getSyncScopeID(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::isMinSignedConstant(), llvm::isNullConstant(), llvm::MemSDNode::isVolatile(), LHS, lowerAtomicArithWithLOCK(), llvm::ISD::MEMBARRIER, llvm::ISD::MERGE_VALUES, N, RHS, llvm::SequentiallyConsistent, and llvm::SyncScope::System.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 31647 of file X86ISelLowering.cpp.
References llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::ATOMIC_LOAD_OR, llvm::ISD::ATOMIC_LOAD_SUB, llvm::ISD::ATOMIC_LOAD_XOR, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getVTList(), llvm::X86ISD::LADD, llvm::X86ISD::LAND, llvm_unreachable, llvm::X86ISD::LOR, llvm::X86ISD::LSUB, llvm::X86ISD::LXOR, and N.
Referenced by combineSetCCAtomicArith(), and lowerAtomicArith().
|
static |
Definition at line 28243 of file X86ISelLowering.cpp.
References DL, llvm::X86Subtarget::hasInt256(), llvm::MVT::is256BitVector(), and splitVectorIntBinary().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 9214 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getFreeze(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::SDValue::hasOneUse(), llvm::Hi, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isFreezeUndef(), llvm::SDValue::isUndef(), llvm::Lo, and llvm::ArrayRef< T >::slice().
Referenced by LowerCONCAT_VECTORS().
|
static |
Definition at line 20150 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), hasIdenticalHalvesShuffleMask(), llvm::X86Subtarget::hasInt256(), llvm::MVT::isVector(), splitVectorIntUnary(), and llvm::ISD::ZERO_EXTEND.
Referenced by LowerANY_EXTEND(), and LowerZERO_EXTEND().
|
static |
Definition at line 31149 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getPMOVMSKB(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::Hi, llvm::MVT::isScalarInteger(), llvm::MVT::isVector(), llvm::Lo, llvm::X86ISD::MOVDQ2Q, llvm::ISD::SCALAR_TO_VECTOR, and llvm::SelectionDAG::SplitScalar().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 31504 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITREVERSE, llvm::ISD::BSWAP, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getGFNICtrlMask(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86ISD::GF2P8AFFINEQB, llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isVector(), llvm::Lo, LowerBITREVERSE_XOP(), Matrix, llvm::ISD::OR, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, splitVectorIntUnary(), and llvm::ISD::SRL.
Referenced by llvm::X86TargetLowering::LowerOperation(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Definition at line 31459 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITREVERSE, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::isVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, splitVectorIntUnary(), and llvm::X86ISD::VPPERM.
Referenced by LowerBITREVERSE().
|
static |
Definition at line 8728 of file X86ISelLowering.cpp.
References createVariablePermute(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), Idx, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 7618 of file X86ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::MVT::changeVectorElementType(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
|
static |
Definition at line 7632 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::CONCAT_VECTORS, Cond, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::getSelect(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::Hi_32(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::Lo_32(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), Select, and llvm::ISD::SETCC.
|
static |
Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which uses: a.
A single scalar load, or a constant. b. Repeated pattern of constants (e.g. <0,1,0,1> or <0,1,2,3,0,1,2,3>).
The VBROADCAST node is returned when a pattern is found, or SDValue() otherwise.
Definition at line 7278 of file X86ISelLowering.cpp.
References llvm::all_of(), assert(), llvm::ISD::BITCAST, llvm::CallingConv::C, llvm::ISD::Constant, llvm::ISD::ConstantFP, llvm::BitVector::count(), extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), getConstantVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::TargetLoweringBase::getPointerTy(), llvm::BuildVectorSDNode::getRepeatedSequence(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasInt256(), llvm::SDNode::hasNUsesOfValue(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::BuildVectorSDNode::isConstantSplat(), isFoldableUseOfShuffle(), llvm::EVT::isInteger(), llvm::ISD::isNormalLoad(), llvm::isNullConstantOrUndef(), llvm::SDNode::isOnlyUserOf(), llvm::EVT::isVector(), llvm::MachineMemOperand::MOLoad, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SelectionDAG::shouldOptForSize(), llvm::X86ISD::SUBV_BROADCAST_LOAD, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VBROADCASTM, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 6498 of file X86ISelLowering.cpp.
References assert(), DL, llvm::First, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::INSERT_VECTOR_ELT, and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by LowerBuildVectorv16i8(), and LowerBuildVectorv8i16().
|
static |
If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors.
NOTE: Its not in our interest to start make a general purpose vectorizer from this, but enough scalar bit operations are created from the later legalization + scalarization stages to need basic support.
Definition at line 8357 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::any_of(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::TargetLoweringBase::isOperationLegalOrPromote(), LHS, LowerShift(), llvm::ISD::OR, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and llvm::ISD::XOR.
|
static |
Custom lower build_vector of v16i8.
Definition at line 6540 of file X86ISelLowering.cpp.
References assert(), DL, llvm::APInt::extractBits(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getZeroVector(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasSSE41(), I, llvm::ISD::INSERT_VECTOR_ELT, llvm::APInt::isZero(), LowerBuildVectorAsInsert(), llvm::ISD::OR, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SHL, llvm::ISD::TRUNCATE, and llvm::X86ISD::VZEXT_MOVL.
|
static |
Custom lower build_vector of v4i32 or v4f32.
Definition at line 6636 of file X86ISelLowering.cpp.
References assert(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SDValue::getConstantOperandAPInt(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::SDValue::isUndef(), llvm::X86::isZeroNode(), and llvm::X86ISD::MOVDDUP.
|
static |
Custom lower build_vector of v8i16.
Definition at line 6622 of file X86ISelLowering.cpp.
References DL, llvm::X86Subtarget::hasSSE41(), and LowerBuildVectorAsInsert().
|
static |
Definition at line 31082 of file X86ISelLowering.cpp.
References assert(), llvm::X86::COND_E, DL, llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), getSETCC(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::X86ISD::LCMPXCHG_DAG, llvm_unreachable, llvm::ISD::MERGE_VALUES, llvm::size(), and Success.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 9356 of file X86ISelLowering.cpp.
References assert(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::MVT::getVectorElementType(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerAVXCONCAT_VECTORS(), and LowerCONCAT_VECTORSvXi1().
|
static |
Definition at line 9281 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::Hi, Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), llvm::isPowerOf2_32(), llvm::isPowerOf2_64(), llvm::SDValue::isUndef(), llvm::X86ISD::KSHIFTL, llvm::Lo, llvm::Log2_64(), llvm::popcount(), llvm::ArrayRef< T >::slice(), widenMaskVectorType(), and widenSubVector().
Referenced by LowerCONCAT_VECTORS().
|
static |
Definition at line 28056 of file X86ISelLowering.cpp.
References llvm::X86ISD::BSR, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::ISD::CTLZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), llvm::MVT::isVector(), LowerVectorCTLZ(), llvm::ISD::TRUNCATE, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 31372 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::KnownBits::countMinTrailingZeros(), DL, llvm::KnownBits::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SelectionDAG::getShiftAmountOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::MVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), LowerVectorCTPOP(), llvm::ISD::MUL, N, llvm::ISD::SHL, llvm::ISD::SRL, and llvm::ISD::SUB.
|
static |
Definition at line 28095 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BSF, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::ISD::CTTZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::isKnownNeverZero(), and llvm::MVT::isVector().
|
static |
Definition at line 32231 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTPS2PH, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::Hi, llvm::Lo, and llvm::SelectionDAG::SplitVectorOperand().
Referenced by llvm::X86TargetLowering::LowerOperation(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Definition at line 24424 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::append(), assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::CONCAT_VECTORS, extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getFixedSizeInBits(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::Hi, I, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::Lo, llvm::ISD::SETGT, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::SM_SentinelUndef, llvm::X86ISD::VSRAI, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 18459 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::X86ISD::KSHIFTR, and widenMaskVector().
|
static |
Definition at line 17890 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), Idx, llvm::MVT::is128BitVector(), llvm::isNullConstant(), llvm::X86::mayFoldIntoStore(), llvm::X86::mayFoldIntoZeroExtend(), llvm::X86ISD::PEXTRB, llvm::ISD::STORE, and llvm::ISD::TRUNCATE.
|
static |
The only differences between FABS and FNEG are the mask and the logic op.
FNEG also has a folding opportunity for FNEG(FABS(x)).
Definition at line 21831 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FABS, llvm::X86ISD::FAND, llvm::ISD::FNEG, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 21892 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::bitsGT(), llvm::MVT::bitsLT(), llvm::APFloat::clearSign(), llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::APInt::getSignedMaxValue(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::isConstOrConstSplatFP(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), llvm::Or, and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 21961 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86ISD::MOVMSK, and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 28275 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::EVT::changeTypeToInteger(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FMAX, llvm::ISD::FMAXIMUM, llvm::X86ISD::FMIN, llvm::ISD::FMINIMUM, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), llvm::Hi, llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::INSERT_VECTOR_ELT, llvm::SelectionDAG::isKnownNeverNaN(), llvm::SelectionDAG::isKnownNeverZeroFloat(), llvm::EVT::isVector(), llvm::TargetOptions::NoNaNsFPMath, llvm::TargetOptions::NoSignedZerosFPMath, llvm::TargetMachine::Options, llvm::peekThroughBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SETLT, llvm::APInt::setSignBit(), llvm::ISD::SETUO, llvm::ISD::SPLAT_VECTOR, std::swap(), llvm::X86ISD::VFPCLASSS, X, and Y.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 21640 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::CVTPH2PS, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::ISD::INSERT_VECTOR_ELT, and llvm::X86ISD::STRICT_CVTPH2PS.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 21669 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::CVTPS2PH, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValue(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::SCALAR_TO_VECTOR, and llvm::X86ISD::STRICT_CVTPS2PH.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Given a scalar cast to FP with a cast to integer operand (almost an ftrunc), try to vectorize the cast ops.
This will avoid an expensive round-trip between XMM and GPR.
Definition at line 19163 of file X86ISelLowering.cpp.
References llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTTP2SI, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP_TO_SINT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SINT_TO_FP, and X.
|
static |
ISD::FROUND is defined to round to nearest with ties rounding away from 0.
This mode isn't supported in hardware on X86. But as long as we aren't compiling with trapping math, we can emulate this with trunc(X + copysign(nextafter(0.5, 0.0), X)).
Definition at line 21809 of file X86ISelLowering.cpp.
References llvm::APFloat::convert(), llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::ISD::FADD, llvm::ISD::FCOPYSIGN, llvm::ISD::FTRUNC, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::APFloat::next(), and llvm::APFloatBase::rmNearestTiesToEven.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 31861 of file X86ISelLowering.cpp.
References assert(), llvm::CallingConv::C, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::StructType::get(), llvm::FixedVectorType::get(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getExternalSymbol(), llvm::SelectionDAG::getIntPtrConstant(), llvm::TargetLoweringBase::getLibcallName(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getTypeForEVT(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::isTargetDarwin(), llvm::TargetLowering::LowerCallTo(), llvm::ISD::MERGE_VALUES, RetTy, llvm::TargetLowering::CallLoweringInfo::setChain(), llvm::TargetLowering::CallLoweringInfo::setDebugLoc(), and llvm::TargetLowering::CallLoweringInfo::setLibCallee().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 29950 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), DL, llvm::ISD::FSHL, llvm::X86ISD::FSHL, llvm::ISD::FSHR, llvm::X86ISD::FSHR, llvm::SelectionDAG::getAnyExtOrTrunc(), getAVX512Node(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::MVT::getIntegerVT(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getPack(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SelectionDAG::getSplatSourceVector(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), getTargetVShiftNode(), getUnpackh(), getUnpackl(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::Hi, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::X86::isConstantSplat(), llvm::MVT::isVector(), llvm::Lo, llvm::ISD::OR, llvm::ISD::SHL, llvm::SelectionDAG::shouldOptForSize(), splitVectorOp(), llvm::ISD::SRL, supportedVectorShiftWithBaseAmnt(), supportedVectorShiftWithImm(), supportedVectorVarShift(), std::swap(), llvm::ISD::TRUNCATE, llvm::APInt::urem(), llvm::X86Subtarget::useBWIRegs(), useVPTERNLOG(), llvm::X86ISD::VSHLD, llvm::X86ISD::VSHLDV, llvm::X86ISD::VSHLI, llvm::X86ISD::VSHRD, llvm::X86ISD::VSHRDV, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.
|
static |
Compute the horizontal sum of bytes in V for the elements of VT.
Requires V to be a byte vector and VT to be an integer vector type with wider elements than V's type. The width of the elements of VT determines how many bytes of V are summed horizontally to produce each element of the result.
Definition at line 31217 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), High, llvm::Low, llvm::X86ISD::PACKUS, llvm::X86ISD::PSADBW, llvm::ISD::SHL, and llvm::ISD::SRL.
Referenced by LowerVectorCTPOP().
|
static |
Definition at line 19061 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SINT_TO_FP, llvm::ISD::STRICT_SINT_TO_FP, llvm::ISD::STRICT_UINT_TO_FP, and llvm::ISD::UINT_TO_FP.
|
static |
Definition at line 19019 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::MVT::getVectorVT(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SINT_TO_FP, llvm::ISD::STRICT_SINT_TO_FP, llvm::ISD::STRICT_UINT_TO_FP, and llvm::ISD::UINT_TO_FP.
|
static |
Definition at line 18452 of file X86ISelLowering.cpp.
References assert(), and insert1BitVector().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 19210 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::OR, llvm::ISD::SETLT, llvm::ISD::SINT_TO_FP, llvm::ISD::SRL, llvm::ISD::STRICT_FADD, llvm::ISD::STRICT_SINT_TO_FP, llvm::ISD::TokenFactor, and llvm::ISD::TRUNCATE.
Referenced by lowerUINT_TO_FP_vec().
|
static |
Definition at line 26779 of file X86ISelLowering.cpp.
References llvm::X86ISD::AADD, llvm::X86ISD::AAND, Addr, llvm::X86ISD::AESDEC128KL, llvm::X86ISD::AESDEC256KL, llvm::X86ISD::AESDECWIDE128KL, llvm::X86ISD::AESDECWIDE256KL, llvm::X86ISD::AESENC128KL, llvm::X86ISD::AESENC256KL, llvm::X86ISD::AESENCWIDE128KL, llvm::X86ISD::AESENCWIDE256KL, llvm::X86ISD::AOR, assert(), llvm::X86ISD::AXOR, llvm::sampleprof::Base, CC, llvm::X86ISD::CMOV, llvm::X86ISD::CMPCCXADD, llvm::X86::COND_B, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::MachineFrameInfo::CreateStackObject(), DL, EmitMaskedTruncSStore(), EmitTruncSStore(), llvm::X86ISD::ENQCMD, llvm::X86ISD::ENQCMDS, expandIntrinsicWChainHelper(), llvm::GATHER, llvm::GATHER_AVX2, llvm::SDNode::getAsZExtVal(), getAVX2GatherNode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), getGatherNode(), llvm::MachineFunction::getInfo(), llvm::getIntrinsicWithChain(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getMaskedStore(), getMaskNode(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getPrefetchNode(), getReadTimeStampCounter(), llvm::MVT::getScalarSizeInBits(), getScatterNode(), getSETCC(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTruncStore(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::INTR_TYPE_CAST_MMX, llvm::isAllOnesConstant(), llvm::X86::isExtendedSwiftAsyncFrameSupported(), isValid(), llvm::X86ISD::LADD, llvm::X86ISD::LAND, llvm::X86ISD::LBTC, llvm::X86ISD::LBTC_RM, llvm::X86ISD::LBTR, llvm::X86ISD::LBTR_RM, llvm::X86ISD::LBTS, llvm::X86ISD::LBTS_RM, llvm_unreachable, llvm::X86ISD::LOR, llvm::X86ISD::LSUB, llvm::X86ISD::LWPINS, llvm::X86ISD::LXOR, MarkEHGuard(), MarkEHRegistrationNode(), llvm::ISD::MERGE_VALUES, llvm::Offset, llvm::IntrinsicData::Opc0, llvm::IntrinsicData::Opc1, Operation, llvm::PREFETCH, llvm::X86ISD::RDPKRU, llvm::RDPMC, llvm::RDPRU, llvm::RDRAND, llvm::RDSEED, llvm::RDTSC, Results, llvm::SCATTER, llvm::MachineFrameInfo::setAdjustsStack(), llvm::MachineFrameInfo::setFrameAddressIsTaken(), llvm::MachineFrameInfo::setHasCopyImplyingStackAdjustment(), llvm::ISD::SHL, Size, llvm::X86ISD::TESTUI, llvm::X86ISD::TPAUSE, llvm::TRUNCATE_TO_MEM_VI16, llvm::TRUNCATE_TO_MEM_VI32, llvm::TRUNCATE_TO_MEM_VI8, llvm::IntrinsicData::Type, llvm::X86ISD::UMWAIT, llvm::ISD::UNINDEXED, llvm::X86ISD::VTRUNC, llvm::X86ISD::VTRUNCS, llvm::X86ISD::VTRUNCUS, llvm::X86ISD::WRPKRU, llvm::XGETBV, llvm::XTEST, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 23133 of file X86ISelLowering.cpp.
References assert(), CC, llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCSwappedOperands(), llvm::MVT::getVectorElementType(), llvm::ISD::SETLT, and std::swap().
Referenced by LowerVSETCC().
|
static |
Definition at line 24755 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDNode::getNumValues(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::isInteger(), and llvm::MVT::isVector().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 32137 of file X86ISelLowering.cpp.
References assert(), ExtendToType(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::is512BitVector(), llvm::SDValue::isUndef(), llvm::X86ISD::MGATHER, and N.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 28259 of file X86ISelLowering.cpp.
References DL, llvm::X86Subtarget::hasInt256(), llvm::MVT::is256BitVector(), and splitVectorIntBinary().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 32029 of file X86ISelLowering.cpp.
References assert(), ExtendToType(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), N, Select, and llvm::ISD::VSELECT.
|
static |
Definition at line 31966 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, ExtendToType(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTypeToTransformTo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::is512BitVector(), llvm::X86ISD::MSCATTER, and N.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 32095 of file X86ISelLowering.cpp.
References assert(), ExtendToType(), llvm::SelectionDAG::getMaskedStore(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::MVT::is512BitVector(), and N.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 28458 of file X86ISelLowering.cpp.
References A, llvm::ISD::ADD, llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::ISD::ANY_EXTEND, assert(), B, llvm::X86Subtarget::canExtendTo512BW(), llvm::SelectionDAG::computeKnownBits(), llvm::enumerate(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), getPack(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, Idx, llvm::MVT::is256BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isNullConstantOrUndef(), llvm::APInt::isSubsetOf(), llvm::ISD::MUL, llvm::ISD::OR, llvm::X86ISD::PMULUDQ, llvm::SmallVectorTemplateBase< T, bool >::push_back(), splitVectorIntBinary(), llvm::ISD::TRUNCATE, llvm::X86ISD::VPMADDUBSW, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLI, and llvm::KnownBits::Zero.
|
static |
Definition at line 28709 of file X86ISelLowering.cpp.
References A, llvm::ISD::ADD, llvm::ISD::AND, assert(), B, llvm::X86Subtarget::canExtendTo512BW(), Fixup, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), getTargetVShiftByConstNode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is256BitVector(), LowervXi8MulWithUNPCK(), llvm::Mul, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::X86ISD::PMULDQ, llvm::X86ISD::PMULUDQ, llvm::ISD::SETGT, llvm::ISD::SIGN_EXTEND, splitVectorIntBinary(), llvm::ISD::SUB, T1, llvm::ISD::TRUNCATE, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 28816 of file X86ISelLowering.cpp.
References A, B, llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftByConstNode(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasInt256(), llvm::Hi, High, llvm::MVT::isVector(), llvm::Lo, llvm::Low, LowervXi8MulWithUNPCK(), LowerXALUO(), llvm::Mul, llvm::ISD::MUL, llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMULO, splitVector(), llvm::ISD::SRA, llvm::ISD::TRUNCATE, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 31591 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::X86ISD::CMP, llvm::X86::COND_NP, DL, llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getSETCC(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::Lo, llvm::SelectionDAG::MaskedValueIsZero(), llvm::ISD::SRL, llvm::ISD::TRUNCATE, X, llvm::ISD::XOR, llvm::X86ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 32244 of file X86ISelLowering.cpp.
|
static |
Definition at line 26702 of file X86ISelLowering.cpp.
References DL, llvm::SelectionDAG::getMergeValues(), getReadTimeStampCounter(), and Results.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 30152 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::X86ISD::BLENDV, llvm::CallingConv::C, convertShiftLeftToScale(), DL, llvm::SelectionDAG::FoldConstantArithmetic(), llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getGFNICtrlMask(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getPack(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SelectionDAG::getSplatSourceVector(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetVShiftByConstNode(), getTargetVShiftNode(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86ISD::GF2P8AFFINEQB, llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::X86::isConstantSplat(), llvm::SelectionDAG::isSplatValue(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MVT::isVector(), llvm::Lo, llvm::ISD::MUL, llvm::ISD::MULHU, llvm::ISD::OR, llvm::X86ISD::PCMPGT, llvm::X86ISD::PMULUDQ, llvm::ISD::ROTL, llvm::ISD::SHL, splitVectorIntBinary(), llvm::ISD::SRL, llvm::ISD::SUB, supportedVectorShiftWithImm(), supportedVectorVarShift(), llvm::ISD::TRUNCATE, llvm::APInt::urem(), llvm::X86Subtarget::useBWIRegs(), useVPTERNLOG(), llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLI, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 18413 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), insert128BitVector(), llvm::MVT::is128BitVector(), llvm::MVT::isInteger(), llvm::X86::isZeroNode(), and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 29465 of file X86ISelLowering.cpp.
References A, llvm::ISD::ADD, assert(), llvm::X86ISD::BLENDV, llvm::CallingConv::C, llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), canWidenShuffleElements(), convertShiftLeftToScale(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getExtOrTrunc(), llvm::SelectionDAG::getNegative(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::APInt::getSignMask(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getUNDEF(), getUnpackh(), getUnpackl(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::SelectionDAG::isKnownNeverZero(), llvm::MVT::isVector(), llvm::Lo, LowerShiftByScalarImmediate(), LowerShiftByScalarVariable(), llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, llvm::ISD::OR, llvm::X86ISD::PACKUS, llvm::X86ISD::PCMPGT, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), R2, llvm::ISD::SETEQ, llvm::ISD::SETGT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, splitVectorIntBinary(), llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, supportedVectorVarShift(), llvm::ISD::TRUNCATE, llvm::X86ISD::VPSHA, llvm::X86ISD::VPSHL, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 29185 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getFreeze(), getGFNICtrlMask(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getTargetConstant(), getTargetVShiftByConstNode(), getTargetVShiftUniformOpcode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::APInt::getZExtValue(), llvm::X86ISD::GF2P8AFFINEQB, llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE42(), llvm::MVT::is512BitVector(), llvm::X86::isConstantSplat(), llvm_unreachable, llvm::Lower, llvm::X86ISD::PCMPGT, llvm::ISD::SETGT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, supportedVectorShiftWithImm(), llvm::APInt::uge(), llvm::Upper, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, llvm::X86ISD::VSRLI, and llvm::ISD::XOR.
Referenced by LowerShift().
|
static |
Definition at line 29346 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86Subtarget::canExtendTo512BW(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSplatSourceVector(), getTargetVShiftByConstNode(), getTargetVShiftNode(), getTargetVShiftUniformOpcode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, supportedVectorShiftWithBaseAmnt(), and llvm::ISD::XOR.
Referenced by LowerShift().
|
static |
Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount.
TODO: Can this be moved to general expansion code?
Definition at line 19011 of file X86ISelLowering.cpp.
References llvm::TargetLowering::expandShiftParts(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Hi, and llvm::Lo.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Try to emit a blend instruction for a shuffle using bit math.
This is used as a fallback approach when first class blend instructions are unavailable. Currently it is only suitable for integer vectors, but could be generalized for floating point vectors if desirable.
Definition at line 10538 of file X86ISelLowering.cpp.
References llvm::AllOnes, assert(), DL, llvm::SelectionDAG::getAllOnesConstant(), getBitSelect(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getVectorElementType(), llvm::MVT::isInteger(), and Size.
Referenced by lower256BitShuffle(), lower512BitShuffle(), lowerShuffleAsBlend(), lowerV16I8Shuffle(), and lowerV8I16Shuffle().
|
static |
Try to emit a bitmask instruction for a shuffle.
This handles cases where we can model a blend exactly as a bitmask due to one of the inputs being zeroable.
Definition at line 10482 of file X86ISelLowering.cpp.
References llvm::AllOnes, llvm::And, llvm::ISD::AND, DL, llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::SelectionDAG::getAllOnesConstant(), llvm::APFloat::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), and Size.
Referenced by lower256BitShuffle(), lower512BitShuffle(), lowerShuffleAsBlend(), lowerV16I8Shuffle(), lowerV4I32Shuffle(), lowerV64I8Shuffle(), and lowerV8I16Shuffle().
|
static |
Lower shuffle using X86ISD::VROTLI rotations.
Definition at line 11294 of file X86ISelLowering.cpp.
References DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE3(), llvm::MVT::is128BitVector(), matchShuffleAsBitRotate(), llvm::ISD::OR, llvm::X86ISD::VROTLI, llvm::X86ISD::VSHLI, and llvm::X86ISD::VSRLI.
Referenced by lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4I32Shuffle(), lowerV64I8Shuffle(), lowerV8I16Shuffle(), and lowerV8I32Shuffle().
|
static |
Try to emit a blend instruction for a shuffle.
This doesn't do any checks for the availability of instructions for blending these values. It relies on the availability of the X86ISD::BLENDI pattern to be matched in the backend with the type given. What it does check for is that the shuffle mask is a blend, or convertible into a blend with zero.
Definition at line 10645 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BLENDI, llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), getVectorMaskingNode(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, is128BitLaneRepeatedShuffleMask(), llvm::ISD::isNormalLoad(), llvm_unreachable, llvm::Lo, lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), llvm::Masked, matchShuffleAsBlend(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::shouldOptForSize(), llvm::MVT::SimpleTy, llvm::SmallVectorBase< Size_T >::size(), Size, and std::swap().
Referenced by lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV2X128Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().
|
static |
Try to lower as a blend of elements from two inputs followed by a single-input permutation.
This matches the pattern where we can blend elements from two inputs and then reduce the shuffle to a single-input permutation.
Definition at line 10806 of file X86ISelLowering.cpp.
References assert(), canWidenShuffleElements(), DL, llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), and Size.
Referenced by lowerShuffleAsDecomposedShuffleMerge(), lowerV4F32Shuffle(), and lowerV4I64Shuffle().
|
static |
Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used.
Definition at line 13805 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorVT(), is128BitLaneCrossingShuffleMask(), llvm::ISD::OR, llvm::X86ISD::PSHUFB, and Size.
Referenced by lowerV16I8Shuffle(), lowerV64I8Shuffle(), and lowerV8I16Shuffle().
|
static |
Try to lower broadcast of a single element.
For convenience, this code also bundles all of the subtarget feature set filtering. While a little annoying to re-dispatch on type here, there isn't a convenient way to factor it out.
Definition at line 12485 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, DL, extract128BitVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::getSplatIndex(), llvm::MVT::getStoreSize(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE3(), Idx, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isInteger(), llvm::ISD::isNormalLoad(), isShuffleFoldableLoad(), lowerShuffleAsTruncBroadcast(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::X86ISD::MOVDDUP, llvm::Offset, llvm::peekThroughBitcasts(), llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VBROADCAST_LOAD.
Referenced by lower512BitShuffle(), lowerV16I16Shuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV8F16Shuffle(), lowerV8F32Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerVECTOR_SHUFFLE().
|
static |
Definition at line 11445 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is512BitVector(), isNoopShuffleMask(), llvm::Lo, matchShuffleAsByteRotate(), llvm::ISD::OR, llvm::X86ISD::PALIGNR, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.
Referenced by lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV2I64Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().
|
static |
Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then permuting the elements of the result in place.
Definition at line 11039 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, is128BitLaneCrossingShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::Lo, llvm::X86ISD::PALIGNR, and llvm::SM_SentinelUndef.
Referenced by lowerShuffleAsDecomposedShuffleMerge(), lowerV16I8Shuffle(), and lowerV64I8Shuffle().
|
static |
Try to lower a vector shuffle as a byte shift sequence.
Definition at line 11553 of file X86ISelLowering.cpp.
References assert(), llvm::APInt::countl_one(), llvm::APInt::countr_one(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::is128BitVector(), isNoopShuffleMask(), isSequentialOrUndefInRange(), isUndefOrInRange(), llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.
Referenced by lowerV16I8Shuffle(), and lowerV8I16Shuffle().
|
static |
Generic routine to decompose a shuffle and blend into independent blends and permutes.
This matches the extremely common pattern for handling combined shuffle+blend operations on newer X86 ISAs where we have very fast blend operations. It will try to pick the best arrangement of shuffles and blends. For vXi8/vXi16 shuffles we may use unpack instead of blend.
Definition at line 11156 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::assign(), DL, llvm::enumerate(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), I, isBroadcastShuffleMask(), isNoopOrBroadcastShuffleMask(), isNoopShuffleMask(), isSingleElementRepeatedMask(), lowerShuffleAsBlendAndPermute(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsUNPCKAndPermute(), llvm::X86::mayFoldLoad(), and llvm::X86ISD::VBROADCAST.
Referenced by lowerShuffleAsSplitOrBlend(), lowerV16I8Shuffle(), lowerV2I64Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV8F32Shuffle(), lowerV8I16Shuffle(), and lowerV8I32Shuffle().
|
static |
Try to lower insertion of a single element into a zero vector.
This is a common pattern that we have especially efficient patterns to lower across all subtarget feature sets.
Definition at line 12216 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), DL, llvm::find_if(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), getScalarValueForVectorElement(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::APInt::getZero(), llvm::MVT::is128BitVector(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), isSoftF16(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSS, llvm::ISD::OR, llvm::ISD::SCALAR_TO_VECTOR, Size, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VZEXT_MOVL, and llvm::ISD::ZERO_EXTEND.
Referenced by lower256BitShuffle(), lower512BitShuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV4F32Shuffle(), lowerV4I32Shuffle(), lowerV8F16Shuffle(), and lowerV8I16Shuffle().
|
static |
Definition at line 12767 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::X86ISD::INSERTPS, and matchShuffleAsInsertPS().
Referenced by lowerV4F32Shuffle().
|
static |
Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane permutation.
This is mainly for cases where we can have non-repeating permutes in each lane.
TODO: This is very similar to lowerShuffleAsLanePermuteAndRepeatedMask, we should investigate merging them.
Attempts to find a sublane permute with the given size that gets all elements into their target lanes.
If successful, fills CrossLaneMask and InLaneMask and returns true. If unsuccessful, returns false and may overwrite InLaneMask.
Definition at line 14758 of file X86ISelLowering.cpp.
References DL, llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), isSequentialOrUndefInRange(), isUndefOrEqual(), llvm::narrowShuffleMaskElts(), and llvm::SM_SentinelUndef.
Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), and lowerV64I8Shuffle().
|
static |
Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane.
This attempts to create a repeated lane shuffle where each lane uses one or two of the lanes of the inputs. The lanes of the input vectors are shuffled in one or two independent shuffles to get the lanes into the position needed by the final shuffle.
Definition at line 15060 of file X86ISelLowering.cpp.
References assert(), llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getVectorShuffle(), is128BitLaneRepeatedShuffleMask(), llvm::M1(), and std::swap().
Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().
|
static |
Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one source with a lane permutation.
This lowering strategy results in four instructions in the worst case for a single-input cross lane shuffle which is lower than any other fully general cross-lane shuffle strategy I'm aware of. Special cases for each particular shuffle pattern should be handled prior to trying this lowering.
Definition at line 14881 of file X86ISelLowering.cpp.
References llvm::all_of(), assert(), computeInLaneShuffleMask(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::MVT::isFloatingPoint(), lowerShuffleAsLanePermuteAndSHUFP(), Size, and splitAndLowerShuffle().
Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), and lowerV8F32Shuffle().
|
static |
Definition at line 14722 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVectorShuffle(), LHS, RHS, and llvm::X86ISD::SHUFP.
Referenced by lowerShuffleAsLanePermuteAndShuffle(), and lowerV4F64Shuffle().
|
static |
Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction.
This specifically targets cases where we end up with alternating between the two inputs, and so can permute them into something that feeds a single UNPCK instruction. Note that this routine only targets integer vectors because for floating point vectors we have a generalized SHUFPS lowering strategy that handles everything that doesn't exactly match an unpack, making this clever lowering unnecessary.
Definition at line 10928 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isFloatingPoint(), isNoopShuffleMask(), Size, llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by lowerShuffleAsDecomposedShuffleMerge(), lowerV16I8Shuffle(), lowerV4I32Shuffle(), and lowerV8I16Shuffle().
|
static |
Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes.
Definition at line 15435 of file X86ISelLowering.cpp.
References assert(), DL, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), Idx, is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), isUndefOrInRange(), llvm::M1(), and llvm::SM_SentinelUndef.
Referenced by lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().
|
static |
Definition at line 11702 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::MVT::getVectorNumElements(), llvm::TargetLoweringBase::isTypeLegal(), matchShuffleAsShift(), Size, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.
Referenced by lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV2I64Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().
|
static |
Lower a vector shuffle as a zero or any extension.
Given a specific number of elements, element bit width, and extension stride, produce either a zero or any extension based on the available features of the subtarget. The extended elements are consecutive and begin and can start from an offsetted element index in the input; to avoid excess shuffling the offset must either being in the bottom lane or at the start of a higher lane. All extended elements must be from the same lane.
Definition at line 11890 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), DL, llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getEXTEND_VECTOR_INREG(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, Idx, llvm::MVT::is128BitVector(), isUndefUpperHalf(), llvm::Lo, llvm::Offset, llvm::X86ISD::PSHUFB, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::ISD::ZERO_EXTEND.
Referenced by lowerShuffleAsZeroOrAnyExtend().
|
static |
Either split a vector in halves or decompose the shuffles and the blend/unpack.
This is provided as a good fallback for many lowerings of non-single-input shuffles with more than one 128-bit lane. In those cases, we want to select between splitting the shuffle into 128-bit components and stitching those back together vs. extracting the single-input shuffles and blending those results.
Definition at line 14668 of file X86ISelLowering.cpp.
References assert(), llvm::SmallBitVector::count(), llvm::count(), DL, llvm::MVT::getSizeInBits(), lowerShuffleAsDecomposedShuffleMerge(), llvm::SmallBitVector::resize(), Size, and splitAndLowerShuffle().
Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV4F64Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().
|
static |
Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0
with larger elements.
This assumes we have AVX2.
Definition at line 12342 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX2(), llvm::MVT::isInteger(), llvm::MVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::X86ISD::VBROADCAST.
Referenced by lowerShuffleAsBroadcast().
|
static |
Try to lower as an unpack of elements from two inputs followed by a single-input permutation.
This matches the pattern where we can unpack elements from two inputs and then reduce the shuffle to a single-input (wider) permutation.
Definition at line 10845 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), llvm::Hi, isUndefOrInRange(), llvm::Lo, llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by lowerShuffleAsDecomposedShuffleMerge().
|
static |
Try to lower a vector shuffle as a dword/qword rotation.
AVX512 has a VALIGND/VALIGNQ instructions that will do an arbitrary rotation of the concatenation of two vectors; This routine will try to generically lower a vector shuffle through such an pattern.
Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.
Definition at line 11502 of file X86ISelLowering.cpp.
References assert(), llvm::APInt::countl_one(), llvm::APInt::countr_one(), DL, llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getTargetConstant(), getZeroVector(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), isSequentialOrUndefInRange(), llvm::Lo, llvm::Low, matchShuffleAsElementRotate(), and llvm::X86ISD::VALIGN.
Referenced by lowerV16I32Shuffle(), lowerV2I64Shuffle(), lowerV4I64Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().
|
static |
Definition at line 10208 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::APInt::extractBits(), getAVX512TruncNode(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::APInt::isAllOnes(), llvm::ISD::isNormalLoad(), isSequentialOrUndefInRange(), isUndefInRange(), llvm::Lo, llvm::Offset, and llvm::X86ISD::VSRLI.
Referenced by lowerV16I16Shuffle(), lowerV16I8Shuffle(), lowerV32I8Shuffle(), and lowerV8I16Shuffle().
|
static |
Definition at line 15706 of file X86ISelLowering.cpp.
References assert(), llvm::APInt::countl_one(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::INSERT_SUBVECTOR, isSequentialOrUndefInRange(), and llvm::X86ISD::VTRUNC.
Referenced by lowerV32I8Shuffle().
|
static |
Try to lower a vector shuffle as a zero extension on any microarch.
This routine will try to do everything in its power to cleverly lower a shuffle which happens to match the pattern of a zero extend. It doesn't check for the profitability of this lowering, it tries to aggressively match this pattern. It will use all of the micro-architectural details it can to emit an efficient lowering. It handles both blends with all-zero inputs to explicitly zero-extend and undef-lanes (sometimes undef due to masking out later).
The reason we have dedicated lowering for zext-style shuffles is that they are both incredibly common and often quite performance sensitive.
Definition at line 12054 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), isSequentialOrUndefInRange(), llvm::Lower, lowerShuffleAsSpecificZeroOrAnyExtend(), llvm::Offset, and llvm::X86ISD::VZEXT_MOVL.
Referenced by lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4I32Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), lowerV8I16Shuffle(), and lowerV8I32Shuffle().
|
static |
If we are extracting two 128-bit halves of a vector and shuffling the result, match that to a 256-bit AVX2 vperm* instruction to avoid a multi-shuffle lowering.
Definition at line 12431 of file X86ISelLowering.cpp.
References llvm::SmallVectorImpl< T >::append(), assert(), llvm::ShuffleVectorSDNode::commuteMask(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandAPInt(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::hasOneUse(), is128BitUnpackShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), and isSingleSHUFPSMask().
Referenced by lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV4F32Shuffle(), and lowerV4I32Shuffle().
|
static |
Definition at line 15754 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::MVT::getVectorNumElements(), I, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::SmallVectorBase< Size_T >::size(), Size, llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::SDNode::uses(), llvm::ISD::VECTOR_SHUFFLE, and llvm::X86ISD::VPERM2X128.
Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().
|
static |
Definition at line 9894 of file X86ISelLowering.cpp.
References assert(), DL, llvm::X86ISD::EXPAND, llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), getMaskNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getZeroVector(), and isNonZeroElementsInOrder().
Referenced by lowerV16F32Shuffle(), lowerV16I32Shuffle(), lowerV4F64Shuffle(), lowerV4I64Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().
|
static |
Definition at line 10433 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::Log2_32(), matchShuffleWithPACK(), and llvm::X86ISD::PACKSS.
Referenced by lowerV16I16Shuffle(), lowerV16I8Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV64I8Shuffle(), and lowerV8I16Shuffle().
|
static |
Definition at line 14103 of file X86ISelLowering.cpp.
References llvm::MVT::changeTypeToInteger(), DL, extractSubVector(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::is512BitVector(), llvm::X86ISD::VPERMV, llvm::X86ISD::VPERMV3, and widenSubVector().
Referenced by combineX86ShuffleChain(), lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV64I8Shuffle(), lowerV8F64Shuffle(), and lowerV8I64Shuffle().
|
static |
Try to lower a shuffle with a single PSHUFB of V1 or V2.
Definition at line 9836 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::X86ISD::PSHUFB, and Size.
Referenced by lowerV16I16Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), and lowerV64I8Shuffle().
|
static |
Definition at line 15679 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), getZeroVector(), matchShuffleWithSHUFPD(), and llvm::X86ISD::SHUFP.
Referenced by lowerV4F64Shuffle(), and lowerV8F64Shuffle().
|
static |
Lower a vector shuffle using the SHUFPS instruction.
This is a helper routine dedicated to lowering vector shuffles using SHUFPS. It makes no assumptions about whether this is the best lowering, it simply uses it.
Definition at line 12974 of file X86ISelLowering.cpp.
References llvm::ShuffleVectorSDNode::commuteMask(), llvm::count_if(), DL, llvm::find_if(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), lowerShuffleWithSHUFPS(), llvm::X86ISD::SHUFP, and std::swap().
Referenced by lowerShuffleWithSHUFPS(), lowerV16F32Shuffle(), lowerV16I32Shuffle(), lowerV4F32Shuffle(), lowerV8F32Shuffle(), and lowerV8I32Shuffle().
|
static |
Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
Definition at line 11863 of file X86ISelLowering.cpp.
References DL, llvm::X86ISD::EXTRQI, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::X86ISD::INSERTQI, matchShuffleAsEXTRQ(), and matchShuffleAsINSERTQ().
Referenced by lowerV16I8Shuffle(), and lowerV8I16Shuffle().
|
static |
Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
This allows for fast cases such as subvector extraction/insertion or shuffling smaller vector types which can lower more efficiently.
Definition at line 15324 of file X86ISelLowering.cpp.
References assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::MVT::getHalfNumVectorElementsVT(), getHalfShuffleMask(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getShuffleHalfVectors(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::Hi, llvm::ISD::INSERT_SUBVECTOR, is128BitUnpackShuffleMask(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isSequentialOrUndefInRange(), isSingleSHUFPSMask(), isUndefLowerHalf(), isUndefUpperHalf(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by lower256BitShuffle(), and lower512BitShuffle().
|
static |
Definition at line 10006 of file X86ISelLowering.cpp.
References llvm::ShuffleVectorSDNode::commuteMask(), llvm::createUnpackShuffleMask(), DL, llvm::SelectionDAG::getNode(), isShuffleEquivalent(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by lowerV16F32Shuffle(), lowerV16I16Shuffle(), lowerV16I32Shuffle(), lowerV16I8Shuffle(), lowerV2F64Shuffle(), lowerV2I64Shuffle(), lowerV32I16Shuffle(), lowerV32I8Shuffle(), lowerV4F32Shuffle(), lowerV4F64Shuffle(), lowerV4I32Shuffle(), lowerV4I64Shuffle(), lowerV64I8Shuffle(), lowerV8F32Shuffle(), lowerV8F64Shuffle(), lowerV8I16Shuffle(), lowerV8I32Shuffle(), and lowerV8I64Shuffle().
|
static |
Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit) followed by unpack 256-bit.
Definition at line 10033 of file X86ISelLowering.cpp.
References llvm::createSplat2ShuffleMask(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), isShuffleEquivalent(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by lowerV16I16Shuffle(), lowerV32I8Shuffle(), and lowerV8I32Shuffle().
|
static |
Definition at line 10158 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::APInt::extractBits(), getAVX512TruncNode(), llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::APInt::isAllOnes(), isSequentialOrUndefInRange(), isUndefInRange(), llvm::peekThroughBitcasts(), and llvm::ISD::TRUNCATE.
Referenced by lowerV16I8Shuffle(), and lowerV8I16Shuffle().
|
static |
Definition at line 24547 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasInt256(), llvm::MVT::isVector(), LowerSIGN_EXTEND_Mask(), llvm::ISD::SIGN_EXTEND_VECTOR_INREG, and splitVectorIntUnary().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 24352 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is512BitVector(), SplitAndExtendv16i1(), and llvm::ISD::TRUNCATE.
Referenced by LowerANY_EXTEND(), and LowerSIGN_EXTEND().
|
static |
Definition at line 24668 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::StoreSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::MachineMemOperand::getFlags(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTypeAction(), llvm::TargetLoweringBase::getTypeToTransformTo(), llvm::SelectionDAG::getUNDEF(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is256BitVector(), llvm::MVT::is32BitVector(), llvm::MVT::is64BitVector(), isFreeToSplitVector(), llvm::MVT::isInteger(), llvm::StoreSDNode::isTruncatingStore(), llvm::EVT::isVector(), splitVectorStore(), llvm::ISD::TRUNCATE, llvm::TargetLoweringBase::TypeWidenVector, and llvm::X86ISD::VEXTRACT_STORE.
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or X86ISD::FMSUBADD node.
Definition at line 8066 of file X86ISelLowering.cpp.
References llvm::Add, llvm::X86ISD::ADDSUB, DL, llvm::ISD::FADD, llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMSUBADD, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::MVT::is512BitVector(), isAddSubOrSubAdd(), and isFMAddSubOrFMSubAdd().
|
static |
Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
Definition at line 8233 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::count_if(), DL, ExpandHorizontalBinOp(), llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::X86ISD::FHSUB, llvm::ISD::FSUB, getHopForBuildVector(), llvm::SDNode::getOperand(), llvm::SDNode::getSimpleValueType(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::HADD, llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSSE3(), llvm::X86ISD::HSUB, llvm::MVT::is256BitVector(), isHopBuildVector(), isHorizontalBinOpPart(), llvm::SDValue::isUndef(), llvm::SDNode::isUndef(), llvm::SDNode::op_values(), and llvm::ISD::SUB.
|
static |
Definition at line 18784 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::PointerType::get(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getEntryNode(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachinePointerInfo::getGOT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::Constant::getNullValue(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), llvm::SDNode::getValueType(), llvm::X86ISD::GlobalBaseReg, llvm::TLSModel::InitialExec, is64Bit(), llvm_unreachable, llvm::TLSModel::LocalExec, llvm::X86II::MO_GOTNTPOFF, llvm::X86II::MO_GOTTPOFF, llvm::X86II::MO_INDNTPOFF, llvm::X86II::MO_NTPOFF, llvm::X86II::MO_TPOFF, llvm::Offset, Ptr, llvm::X86ISD::Wrapper, and llvm::X86ISD::WrapperRIP.
|
static |
Definition at line 18716 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getNode(), GetTLSADDR(), llvm::SDValue::getValue(), llvm::X86ISD::GlobalBaseReg, and llvm::X86II::MO_TLSGD.
|
static |
Definition at line 18730 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getEntryNode(), GetTLSADDR(), and llvm::X86II::MO_TLSGD.
|
static |
Definition at line 18738 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::getEntryNode(), GetTLSADDR(), and llvm::X86II::MO_TLSGD.
|
static |
Definition at line 18744 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::sampleprof::Base, llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getEntryNode(), llvm::GlobalAddressSDNode::getGlobal(), llvm::MachineFunction::getInfo(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::SelectionDAG::getTargetGlobalAddress(), GetTLSADDR(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::X86ISD::GlobalBaseReg, llvm::X86MachineFunctionInfo::incNumLocalDynamicTLSAccesses(), llvm::X86II::MO_DTPOFF, llvm::X86II::MO_TLSLD, llvm::X86II::MO_TLSLDM, llvm::Offset, and llvm::X86ISD::Wrapper.
|
static |
Definition at line 20617 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::CONCAT_VECTORS, DL, extract128BitVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::Hi, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::Lo, llvm::ISD::SETGT, llvm::ISD::SETNE, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, and llvm::ISD::TRUNCATE.
|
static |
This function lowers a vector truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations.
Definition at line 20566 of file X86ISelLowering.cpp.
References DL, llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::isPowerOf2_32(), isUpperSubvectorUndef(), llvm::Lo, LowerTruncateVecPack(), llvm::ISD::TRUNCATE, truncateVectorWithPACKSS(), truncateVectorWithPACKUS(), and widenSubVector().
Referenced by LowerTruncateVecPack().
|
static |
This function lowers a vector truncation of 'extended sign-bits' or 'extended zero-bits' values.
vXi16/vXi32/vXi64 to vXi8/vXi16/vXi32 into X86ISD::PACKSS/PACKUS operations.
Definition at line 20532 of file X86ISelLowering.cpp.
References DL, llvm::MVT::getHalfNumVectorElementsVT(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), isUpperSubvectorUndef(), llvm::Lo, LowerTruncateVecPackWithSignBits(), matchTruncateWithPACK(), truncateVectorWithPACK(), and widenSubVector().
Referenced by LowerTruncateVecPackWithSignBits().
|
static |
32-bit unsigned integer to float expansion.
Definition at line 19556 of file X86ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getFPExtendOrRound(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), getShuffleVectorZeroOrUndef(), llvm::SelectionDAG::getStrictFPExtendOrRound(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::Or, llvm::ISD::OR, llvm::ISD::SCALAR_TO_VECTOR, and llvm::ISD::STRICT_FSUB.
|
static |
64-bit unsigned integer to double expansion.
Definition at line 19488 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::X86ISD::FHADD, llvm::ISD::FSUB, llvm::ConstantVector::get(), llvm::ConstantDataVector::get(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetLoweringInfo(), getUnpackl(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasSSE3(), llvm::APFloatBase::IEEEdouble(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SCALAR_TO_VECTOR, and shouldUseHorizontalOp().
|
static |
Definition at line 19606 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTUI2P, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::X86Subtarget::hasAVX512(), llvm::Or, llvm::ISD::OR, llvm::X86ISD::STRICT_CVTUI2P, llvm::ISD::STRICT_FSUB, and llvm::ISD::ZERO_EXTEND.
Referenced by lowerUINT_TO_FP_vec().
|
static |
Definition at line 19817 of file X86ISelLowering.cpp.
References llvm::SDValue::getSimpleValueType(), llvm_unreachable, lowerINT_TO_FP_vXi64(), lowerUINT_TO_FP_v2i32(), lowerUINT_TO_FP_vXi32(), and llvm::MVT::SimpleTy.
|
static |
Definition at line 19659 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::X86ISD::BLENDI, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::FADD, llvm::ISD::FSUB, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::TargetLoweringBase::getPointerTy(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE41(), High, llvm::APFloatBase::IEEEdouble(), llvm::APFloatBase::IEEEsingle(), llvm::ISD::INSERT_SUBVECTOR, llvm::Low, llvm::MachineMemOperand::MOLoad, llvm::Or, llvm::ISD::OR, llvm::ISD::SRL, llvm::ISD::STRICT_FADD, llvm::ISD::STRICT_FSUB, llvm::ISD::STRICT_UINT_TO_FP, llvm::ISD::UINT_TO_FP, llvm::X86ISD::VBROADCAST_LOAD, and llvm::ISD::ZERO_EXTEND.
Referenced by lowerUINT_TO_FP_vec().
|
static |
Handle lowering of 16-lane 32-bit floating point shuffles.
Definition at line 16801 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), lowerShuffleAsBlend(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleToEXPAND(), lowerShuffleWithPERMV(), lowerShuffleWithSHUFPS(), lowerShuffleWithUNPCK(), llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::SmallVectorBase< Size_T >::size(), llvm::X86ISD::VPERMILPI, and llvm::X86ISD::VPERMILPV.
Referenced by lower512BitShuffle().
|
static |
Handle lowering of 16-lane 16-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v16i16 shuffling..
Definition at line 16324 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsSplitOrBlend(), lowerShuffleAsVTRUNC(), lowerShuffleAsZeroOrAnyExtend(), lowerShufflePairAsUNPCKAndPermute(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithPSHUFB(), lowerShuffleWithUNPCK(), lowerShuffleWithUNPCK256(), and lowerV8I16GeneralSingleInputShuffle().
Referenced by lower256BitShuffle().
|
static |
Handle lowering of 16-lane 32-bit integer shuffles.
Definition at line 16941 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), is128BitLaneRepeatedShuffleMask(), isSingleSHUFPSMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsByteRotate(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsVALIGN(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleToEXPAND(), lowerShuffleWithPERMV(), lowerShuffleWithSHUFPS(), lowerShuffleWithUNPCK(), llvm::X86ISD::PSHUFD, and llvm::SmallVectorBase< Size_T >::size().
Referenced by lower512BitShuffle().
|
static |
Generic lowering of v16i8 shuffles.
This is a hybrid strategy to lower v16i8 vectors. It first attempts to detect any complexity reducing interleaving. If that doesn't help, it uses UNPCK to spread the i8 elements across two i16-element vectors, and uses the existing lowering for v8i16 blends on each half, finally PACK-ing them back together.
Definition at line 14147 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::array_pod_sort(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), canLowerByDroppingElements(), llvm::copy_if(), llvm::count_if(), DL, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorShuffle(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), I, lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBlendOfPSHUFBs(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsByteShiftMask(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsShift(), lowerShuffleAsVTRUNC(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithSSE4A(), lowerShuffleWithUNPCK(), lowerShuffleWithVPMOV(), llvm::Masked, llvm::none_of(), llvm::X86ISD::PACKUS, llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), llvm::unique(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, llvm::X86ISD::VPPERM, and llvm::X86ISD::VSRLI.
Referenced by lower128BitShuffle().
|
static |
Handle lowering of 2-lane 64-bit floating point shuffles.
This is the basis function for the 2-lane 64-bit shuffles as we have full support for floating point shuffles but not integer shuffles. These instructions will incur a domain crossing penalty on some chips though so it is better to avoid lowering through this for integer vectors where possible.
Definition at line 12790 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), getScalarValueForVectorElement(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), isShuffleEquivalent(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsElementInsertion(), lowerShuffleOfExtractsAsVperm(), lowerShuffleWithUNPCK(), llvm::X86ISD::MOVSD, llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::SHUFP, llvm::SM_SentinelUndef, and llvm::X86ISD::VPERMILPI.
Referenced by lower128BitShuffle().
|
static |
Handle lowering of 2-lane 64-bit integer shuffles.
Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by the integer unit to minimize domain crossing penalties. However, for blends it falls back to the floating point shuffle operation with appropriate bit casting.
Definition at line 12874 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsShift(), lowerShuffleAsVALIGN(), lowerShuffleOfExtractsAsVperm(), lowerShuffleWithUNPCK(), and llvm::X86ISD::PSHUFD.
Referenced by lower128BitShuffle().
|
static |
Handle lowering 2-lane 128-bit shuffles.
Definition at line 14940 of file X86ISelLowering.cpp.
References assert(), canWidenShuffleElements(), DL, llvm::ISD::EXTRACT_SUBVECTOR, getBROADCAST_LOAD(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getStoreSize(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorAllZeros(), isShuffleEquivalent(), lowerShuffleAsBlend(), llvm::X86::mayFoldLoad(), llvm::peekThroughBitcasts(), llvm::peekThroughOneUseBitcasts(), llvm::X86ISD::SHUF128, llvm::X86ISD::SUBV_BROADCAST_LOAD, and llvm::X86ISD::VPERM2X128.
Referenced by lowerV4F64Shuffle(), and lowerV4I64Shuffle().
|
static |
Handle lowering of 32-lane 16-bit integer shuffles.
Definition at line 17038 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SDValue::getSimpleValueType(), is128BitLaneRepeatedShuffleMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsByteRotate(), lowerShuffleAsShift(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithPSHUFB(), lowerShuffleWithUNPCK(), and lowerV8I16GeneralSingleInputShuffle().
Referenced by lower512BitShuffle().
|
static |
Handle lowering of 32-lane 8-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v32i8 shuffling..
Definition at line 16447 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), is128BitLaneCrossingShuffleMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsSplitOrBlend(), lowerShuffleAsVTRUNC(), lowerShuffleAsVTRUNCAndUnpack(), lowerShuffleAsZeroOrAnyExtend(), lowerShufflePairAsUNPCKAndPermute(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithPSHUFB(), lowerShuffleWithUNPCK(), and lowerShuffleWithUNPCK256().
Referenced by lower256BitShuffle().
|
static |
Lower 4-lane 32-bit floating point shuffles.
Uses instructions exclusively from the floating point unit to minimize domain crossing penalties, as these are sufficient to implement all v4f32 shuffles.
Definition at line 13066 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), isShuffleEquivalent(), isSingleSHUFPSMask(), lowerShuffleAsBlend(), lowerShuffleAsBlendAndPermute(), lowerShuffleAsBroadcast(), lowerShuffleAsElementInsertion(), lowerShuffleAsInsertPS(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleOfExtractsAsVperm(), lowerShuffleWithSHUFPS(), lowerShuffleWithUNPCK(), llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::X86ISD::SHUFP, and llvm::X86ISD::VPERMILPI.
Referenced by lower128BitShuffle().
|
static |
Handle lowering of 4-lane 64-bit floating point shuffles.
Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2 isn't available.
Definition at line 15827 of file X86ISelLowering.cpp.
References llvm::all_of(), assert(), llvm::ISD::BUILD_VECTOR, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), is128BitLaneCrossingShuffleMask(), isShuffleEquivalent(), isShuffleMaskInputInPlace(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsLanePermuteAndSHUFP(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsSplitOrBlend(), lowerShuffleToEXPAND(), lowerShuffleWithSHUFPD(), lowerShuffleWithUNPCK(), lowerV2X128Shuffle(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by lower256BitShuffle().
|
static |
Lower 4-lane i32 vector shuffles.
We try to handle these with integer-domain shuffles where we can, but for blends we use the floating point domain blend instructions.
Definition at line 13170 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), if(), isShuffleEquivalent(), isSingleSHUFPSMask(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsShift(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleOfExtractsAsVperm(), lowerShuffleWithUNPCK(), llvm::Masked, and llvm::X86ISD::PSHUFD.
Referenced by lower128BitShuffle().
|
static |
Handle lowering of 4-lane 64-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v4i64 shuffling..
Definition at line 15947 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), is128BitLaneRepeatedShuffleMask(), isShuffleMaskInputInPlace(), lowerShuffleAsBlend(), lowerShuffleAsBlendAndPermute(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsVALIGN(), lowerShuffleToEXPAND(), lowerShuffleWithUNPCK(), lowerV2X128Shuffle(), llvm::narrowShuffleMaskElts(), llvm::X86ISD::PSHUFD, and llvm::X86ISD::VPERMI.
Referenced by lower256BitShuffle().
|
static |
Try to lower a vector shuffle as a 128-bit shuffles.
Definition at line 16641 of file X86ISelLowering.cpp.
References assert(), canWidenShuffleElements(), llvm::SmallVectorImpl< T >::clear(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getZeroVector(), insert128BitVector(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is512BitVector(), isShuffleEquivalent(), isUndef(), llvm::narrowShuffleMaskElts(), OpIndex, llvm::X86ISD::SHUF128, and llvm::SmallVectorBase< Size_T >::size().
Referenced by lowerV8F64Shuffle(), and lowerV8I64Shuffle().
|
static |
Handle lowering of 64-lane 8-bit integer shuffles.
Definition at line 17102 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SDValue::getSimpleValueType(), is128BitLaneCrossingShuffleMask(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBlendOfPSHUFBs(), lowerShuffleAsByteRotate(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithPACK(), lowerShuffleWithPERMV(), lowerShuffleWithPSHUFB(), lowerShuffleWithUNPCK(), llvm::Masked, and splitAndLowerShuffle().
Referenced by lower512BitShuffle().
|
static |
Lower 8-lane 16-bit floating point shuffles.
Definition at line 14072 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getVectorShuffle(), lowerShuffleAsBroadcast(), and lowerShuffleAsElementInsertion().
Referenced by lower128BitShuffle().
|
static |
Handle lowering of 8-lane 32-bit floating point shuffles.
Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2 isn't available.
Definition at line 16060 of file X86ISelLowering.cpp.
References assert(), computeInLaneShuffleMask(), DL, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), isUnpackWdShuffleMask(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsSplitOrBlend(), lowerShuffleAsZeroOrAnyExtend(), lowerShufflePairAsUNPCKAndPermute(), lowerShuffleToEXPAND(), lowerShuffleWithSHUFPS(), lowerShuffleWithUNPCK(), llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::SmallVectorBase< Size_T >::size(), splitAndLowerShuffle(), llvm::X86ISD::VPERMILPI, llvm::X86ISD::VPERMILPV, and llvm::X86ISD::VPERMV.
Referenced by lower256BitShuffle().
|
static |
Handle lowering of 8-lane 64-bit floating point shuffles.
Definition at line 16747 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), getV4X86ShuffleImm8ForMask(), is128BitLaneCrossingShuffleMask(), is256BitLaneRepeatedShuffleMask(), isShuffleEquivalent(), lowerShuffleAsBlend(), lowerShuffleToEXPAND(), lowerShuffleWithPERMV(), lowerShuffleWithSHUFPD(), lowerShuffleWithUNPCK(), lowerV4X128Shuffle(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::VPERMI, and llvm::X86ISD::VPERMILPI.
Referenced by lower512BitShuffle().
|
static |
Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part.
The lowering strategy is to try to form pairs of input lanes which are targeted at the same half of the final vector, and then use a dword shuffle to place them onto the right half, and finally unpack the paired lanes into their final position.
The exact breakdown of how to form these dword pairs and align them on the correct sides is really tricky. See the comments within the function for more of the details.
This code also handles repeated 128-bit lanes of v8i16 shuffles, but each lane must shuffle the exact same way. In fact, you must pass a v8 Mask to this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16 vector, form the analogous 128-bit 8-element Mask.
Definition at line 13311 of file X86ISelLowering.cpp.
References llvm::array_pod_sort(), assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::ArrayRef< T >::begin(), llvm::copy_if(), llvm::count(), llvm::count_if(), llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::ArrayRef< T >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::end(), llvm::SmallVectorImpl< T >::erase(), llvm::find(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getV4X86ShuffleImm8ForMask(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::is_contained(), isNoopShuffleMask(), isSequentialOrUndefInRange(), isUndefOrEqual(), isUndefOrInRange(), llvm_unreachable, llvm::lower_bound(), lowerV8I16GeneralSingleInputShuffle(), llvm::M0(), llvm::M1(), llvm::Match, llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorImpl< T >::resize(), llvm::ArrayRef< T >::size(), llvm::SmallVectorBase< Size_T >::size(), std::swap(), and llvm::unique().
Referenced by lowerV16I16Shuffle(), lowerV32I16Shuffle(), lowerV8I16GeneralSingleInputShuffle(), and lowerV8I16Shuffle().
|
static |
Generic lowering of 8-lane i16 shuffles.
This handles both single-input shuffles and combined shuffle/blends with two inputs. The single input shuffles are immediately delegated to a dedicated lowering routine.
The blends are lowered in one of three fundamental ways. If there are few enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle of the input is significantly cheaper when lowered as an interleaving of the two inputs, try to interleave them. Otherwise, blend the low and high halves of the inputs separately (making them have relatively few inputs) and then concatenate them.
Definition at line 13868 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::any_of(), assert(), llvm::X86ISD::BLENDI, canLowerByDroppingElements(), concatSubVectors(), llvm::count_if(), DL, extract128BitVector(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getOpcode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), getZeroVector(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBlendOfPSHUFBs(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsByteShiftMask(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsElementInsertion(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsShift(), lowerShuffleAsVTRUNC(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleWithPACK(), lowerShuffleWithSSE4A(), lowerShuffleWithUNPCK(), lowerShuffleWithVPMOV(), lowerV8I16GeneralSingleInputShuffle(), llvm::Masked, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::peekThroughBitcasts(), llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.
Referenced by lower128BitShuffle().
|
static |
Handle lowering of 8-lane 32-bit integer shuffles.
This routine is only called when we have AVX2 and thus a reasonable instruction set for v8i32 shuffling..
Definition at line 16182 of file X86ISelLowering.cpp.
References assert(), llvm::count_if(), DL, llvm::SelectionDAG::getBitcast(), getConstVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), is128BitLaneRepeatedShuffleMask(), isSingleSHUFPSMask(), isUnpackWdShuffleMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsDecomposedShuffleMerge(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsSplitOrBlend(), lowerShuffleAsVALIGN(), lowerShuffleAsZeroOrAnyExtend(), lowerShufflePairAsUNPCKAndPermute(), lowerShuffleToEXPAND(), lowerShuffleWithSHUFPS(), lowerShuffleWithUNPCK(), lowerShuffleWithUNPCK256(), llvm::X86ISD::PSHUFD, llvm::SmallVectorBase< Size_T >::size(), and llvm::X86ISD::VPERMV.
Referenced by lower256BitShuffle().
|
static |
Handle lowering of 8-lane 64-bit integer shuffles.
Definition at line 16868 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), getV4X86ShuffleImm8ForMask(), is128BitLaneRepeatedShuffleMask(), is256BitLaneRepeatedShuffleMask(), lowerShuffleAsBlend(), lowerShuffleAsByteRotate(), lowerShuffleAsShift(), lowerShuffleAsVALIGN(), lowerShuffleToEXPAND(), lowerShuffleWithPERMV(), lowerShuffleWithUNPCK(), lowerV4X128Shuffle(), llvm::narrowShuffleMaskElts(), llvm::X86ISD::PSHUFD, and llvm::X86ISD::VPERMI.
Referenced by lower512BitShuffle().
|
static |
Definition at line 25135 of file X86ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::expandVACopy(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMemcpy(), llvm::X86Subtarget::isCallingConvWin64(), and llvm::X86Subtarget::isTarget64BitLP64().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Top-level lowering for x86 vector shuffles.
This handles decomposition, canonicalization, and lowering of all x86 vector shuffles. Most of the specific lowering strategies are encapsulated above in helper routines. The canonicalization attempts to widen shuffles to involve fewer lanes of wider elements, consolidate symmetric patterns s.t. only one of the two inputs needs to be tested, etc.
Definition at line 17616 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::any_of(), assert(), canCombineAsMaskOperation(), canonicalizeShuffleMaskWithCommute(), canonicalizeShuffleMaskWithHorizOp(), canWidenShuffleElements(), llvm::ShuffleVectorSDNode::commuteMask(), computeZeroableShuffleElements(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getCommutedVectorShuffle(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::ShuffleVectorSDNode::getMask(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::is_contained(), llvm::APInt::isAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::MVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm_unreachable, lower128BitShuffle(), lower1BitShuffle(), lower256BitShuffle(), lower512BitShuffle(), lowerShuffleAsBroadcast(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelZero, and std::swap().
|
static |
Definition at line 22288 of file X86ISelLowering.cpp.
References llvm::ISD::AND, assert(), CC, llvm::EVT::changeVectorElementType(), llvm::MVT::changeVectorElementType(), llvm::X86ISD::CMP, llvm::SelectionDAG::computeKnownBits(), llvm::X86::COND_E, llvm::X86::COND_NE, DL, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitWidth(), llvm::KnownBits::getConstant(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, llvm::EVT::is512BitVector(), llvm::KnownBits::isConstant(), llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::KnownBits::isZero(), llvm::X86ISD::KORTEST, LHS, llvm::Lo, llvm::X86ISD::MOVMSK, llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PTEST, RHS, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::SelectionDAG::SplitScalar(), llvm::SelectionDAG::SplitVector(), llvm::X86Subtarget::useAVX512Regs(), and llvm::ISD::XOR.
Referenced by combinePredicateReduction(), and MatchVectorAllEqualTest().
|
static |
Definition at line 28034 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512DQ(), DL, llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerVectorCTLZ_AVX512CDI(), LowerVectorCTLZInRegLUT(), and splitVectorIntUnary().
Referenced by LowerCTLZ().
|
static |
Lower a vector CTLZ using native supported vector CTLZ instruction.
Definition at line 27922 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::CTLZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), splitVectorIntUnary(), llvm::ISD::SUB, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerVectorCTLZ().
|
static |
Definition at line 27952 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::Hi, llvm::Lo, llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SETEQ, llvm::ISD::SIGN_EXTEND, and llvm::ISD::SRL.
Referenced by LowerVectorCTLZ().
|
static |
Definition at line 31328 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512DQ(), llvm::ISD::CTPOP, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSSE3(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), LowerHorizontalByteSum(), LowerVectorCTPOPInRegLUT(), splitVectorIntUnary(), llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerCTPOP().
|
static |
Definition at line 31281 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, assert(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::X86ISD::PSHUFB, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::SRL.
Referenced by LowerVectorCTPOP().
|
static |
Try to lower a VSELECT instruction to a vector shuffle.
Definition at line 17758 of file X86ISelLowering.cpp.
References Cond, createShuffleMaskFromVSELECT(), llvm::SelectionDAG::getVectorShuffle(), llvm::ISD::isBuildVectorOfConstantSDNodes(), LHS, and RHS.
|
static |
Definition at line 23246 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::ISD::AND, assert(), llvm::BitWidth, CC, llvm::EVT::changeVectorElementTypeToInteger(), cheapX86FSETCC_SSE(), llvm::X86ISD::CMPM, llvm::X86ISD::CMPP, llvm::SelectionDAG::ComputeNumSignBits(), Cond, EQ, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getFixedSizeInBits(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::APInt::getSignMask(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSE42(), incDecVectorConstant(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isConstantSplatVector(), llvm::isConstOrConstSplat(), llvm::MVT::isFloatingPoint(), llvm::APInt::isMaxSignedValue(), llvm::APInt::isMinSignedValue(), llvm::TargetLoweringBase::isOperationLegal(), llvm::APInt::isPowerOf2(), isSoftF16(), llvm::ISD::isTrueWhenEqual(), llvm::ISD::isUnsignedIntSetCC(), llvm::APInt::isZero(), llvm_unreachable, llvm::APInt::logBase2(), LowerIntVSETCC_AVX512(), LowerVSETCCWithSUBUS(), llvm::ISD::OR, llvm::X86ISD::PCMPEQ, llvm::X86ISD::PCMPGT, llvm::peekThroughBitcasts(), llvm::ISD::SETEQ, llvm::SDNode::setFlags(), llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::SelectionDAG::SignBitIsZero(), splitIntVSETCC(), llvm::ISD::SRA, llvm::X86ISD::STRICT_CMPM, llvm::X86ISD::STRICT_CMPP, llvm::ISD::STRICT_FSETCC, llvm::ISD::STRICT_FSETCCS, std::swap(), llvm::ISD::TokenFactor, translateX86FSETCC(), llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::X86ISD::VPCOM, llvm::X86ISD::VPCOMU, and llvm::ISD::XOR.
|
static |
As another special case, use PSUBUS[BW] when it's profitable.
E.g. for Op0 u<= Op1: t = psubus Op0, Op1 pcmpeq t, <0..0>
Definition at line 23191 of file X86ISelLowering.cpp.
References Cond, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasSSE2(), incDecVectorConstant(), llvm::X86ISD::PCMPEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, std::swap(), and llvm::ISD::USUBSAT.
Referenced by LowerVSETCC().
|
static |
Definition at line 28630 of file X86ISelLowering.cpp.
References A, B, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPack(), getUnpackh(), getUnpackl(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::Low, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::SHL.
Referenced by LowerMULH(), and LowerMULO().
|
static |
Definition at line 50313 of file X86ISelLowering.cpp.
References llvm::ISD::CTLZ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::Log2_32(), and llvm::ISD::SRL.
Referenced by combineOrCmpEqZeroToCtlzSrl().
|
static |
Definition at line 52977 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::X86ISD::FAND, llvm::X86ISD::FANDN, llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SelectionDAG::getBitcast(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::isVector(), llvm_unreachable, N, llvm::ISD::OR, and llvm::ISD::XOR.
Referenced by combineFAnd(), combineFAndn(), and combineFOr().
|
static |
Definition at line 24001 of file X86ISelLowering.cpp.
References assert(), Cond, DL, llvm::SelectionDAG::getNode(), getSETCC(), getX86XALUOOp(), and llvm::ISD::MERGE_VALUES.
|
static |
Definition at line 20282 of file X86ISelLowering.cpp.
References assert(), DL, llvm::MVT::getVectorElementType(), llvm::X86Subtarget::hasAVX(), LowerAVXExtend(), and LowerZERO_EXTEND_Mask().
Referenced by llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 20225 of file X86ISelLowering.cpp.
References assert(), llvm::X86Subtarget::canExtendTo512DQ(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getSelect(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is512BitVector(), llvm::ISD::SIGN_EXTEND, SplitAndExtendv16i1(), llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerZERO_EXTEND().
|
static |
Definition at line 26729 of file X86ISelLowering.cpp.
References llvm::WinEHFuncInfo::EHGuardFrameIndex, llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getWinEHFuncInfo(), and llvm::report_fatal_error().
Referenced by LowerINTRINSIC_W_CHAIN().
|
static |
Definition at line 26711 of file X86ISelLowering.cpp.
References llvm::WinEHFuncInfo::EHRegNodeFrameIndex, llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getWinEHFuncInfo(), and llvm::report_fatal_error().
Referenced by LowerINTRINSIC_W_CHAIN().
|
static |
Definition at line 17315 of file X86ISelLowering.cpp.
References isSequentialOrUndefInRange(), llvm::X86ISD::KSHIFTL, llvm::X86ISD::KSHIFTR, llvm::Left, llvm::Low, and Size.
Referenced by lower1BitShuffle().
Definition at line 58264 of file X86ISelLowering.cpp.
References llvm::StringRef::empty(), llvm::StringRef::find_first_not_of(), llvm::StringRef::starts_with(), and llvm::StringRef::substr().
Referenced by llvm::X86TargetLowering::ExpandInlineAsm().
|
static |
Definition at line 38351 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BLENDI, DL, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), getV4X86ShuffleImm(), llvm::MVT::getVectorVT(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), llvm::Hi, llvm::X86ISD::INSERTPS, llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isAnyZero(), isRepeatedTargetShuffleMask(), isUndefInRange(), isUndefOrInRange(), isUndefOrZeroInRange(), llvm::Lo, llvm::M0(), llvm::M1(), matchShuffleAsBlend(), matchShuffleAsByteRotate(), matchShuffleAsElementRotate(), matchShuffleAsInsertPS(), matchShuffleWithSHUFPD(), llvm::Offset, llvm::X86ISD::PALIGNR, S1, llvm::X86ISD::SHUFP, llvm::SmallVectorBase< Size_T >::size(), llvm::SM_SentinelUndef, and llvm::X86ISD::VALIGN.
Referenced by combineX86ShuffleChain().
|
static |
Definition at line 38155 of file X86ISelLowering.cpp.
References llvm::MVT::changeTypeToInteger(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::APInt::getOneBitSet(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::KnownBits::isAllOnes(), isTargetShuffleEquivalent(), llvm::KnownBits::isZero(), llvm_unreachable, llvm::SelectionDAG::MaskedVectorIsZero(), matchShuffleWithPACK(), matchShuffleWithUNPCK(), llvm::X86ISD::MOVHLPS, llvm::X86ISD::MOVLHPS, llvm::X86ISD::MOVSD, llvm::X86ISD::MOVSH, llvm::X86ISD::MOVSS, llvm::KnownBits::One, llvm::ISD::OR, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::APInt::setBit(), llvm::APInt::setBits(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, std::swap(), llvm::X86ISD::UNPCKH, llvm::X86ISD::UNPCKL, and llvm::KnownBits::Zero.
Referenced by combineX86ShuffleChain().
Definition at line 50219 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::X86ISD::ANDNP, llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), N, llvm::ISD::OR, std::swap(), X, and Y.
Referenced by combineLogicBlendIntoPBLENDV().
|
static |
Definition at line 55470 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, canReduceVMulWidth(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::Mul, llvm::ISD::MUL, SplitOpsAndApply(), std::swap(), llvm::ISD::TRUNCATE, and llvm::X86ISD::VPMADDWD.
Referenced by combineAdd().
|
static |
Definition at line 55578 of file X86ISelLowering.cpp.
References assert(), llvm::EVT::bitsLT(), llvm::ISD::BUILD_VECTOR, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasSSE2(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), llvm::ISD::MUL, llvm::ISD::SIGN_EXTEND, SplitOpsAndApply(), std::swap(), and llvm::X86ISD::VPMADDWD.
Referenced by combineAdd().
|
static |
Helper for matching BINOP(EXTRACTELT(X,0),BINOP(EXTRACTELT(X,1),...)) style scalarized (associative) reduction patterns.
Partial reductions are supported when the pointer SrcMask is non-null. TODO - move this to SelectionDAG?
Definition at line 22219 of file X86ISelLowering.cpp.
References assert(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::begin(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::empty(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::end(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), I, Idx, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::insert(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by combineAnd(), combineOr(), and MatchVectorAllEqualTest().
|
static |
Definition at line 11274 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getIntegerVT(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::ShuffleVectorInst::isBitRotateMask(), and isNoopShuffleMask().
|
static |
Definition at line 10561 of file X86ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::MVT::is256BitVector(), llvm::ISD::isBuildVectorAllZeros(), IsElementEquivalent(), llvm::SDValue::isUndef(), and llvm::SM_SentinelUndef.
Referenced by lowerShuffleAsBlend(), and matchBinaryPermuteShuffle().
|
static |
Try to lower a vector shuffle as a byte rotation.
SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary byte-rotation of the concatenation of two vectors; pre-SSSE3 can use a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will try to generically lower a vector shuffle through such an pattern. It does not check for the profitability of lowering either as PALIGNR or PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form. This matches shuffle vectors that look like:
v8i16 [11, 12, 13, 14, 15, 0, 1, 2]
Essentially it concatenates V1 and V2, shifts right by some number of elements, and takes the low elements as the result. Note that while this is specified as a right shift because x86 is little-endian, it is a left rotate of the vector lanes.
Definition at line 11423 of file X86ISelLowering.cpp.
References is128BitLaneRepeatedShuffleMask(), isAnyZero(), matchShuffleAsElementRotate(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by lowerShuffleAsByteRotate(), and matchBinaryPermuteShuffle().
Try to match a vector shuffle as an element rotation.
This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.
Definition at line 11338 of file X86ISelLowering.cpp.
References assert(), llvm::Hi, llvm::Lo, and llvm::SM_SentinelUndef.
Referenced by lowerShuffleAsVALIGN(), matchBinaryPermuteShuffle(), and matchShuffleAsByteRotate().
|
static |
Definition at line 11741 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), Idx, llvm::APInt::isAllOnes(), isUndefUpperHalf(), Size, and llvm::SM_SentinelUndef.
Referenced by combineX86ShuffleChain(), and lowerShuffleWithSSE4A().
|
static |
Definition at line 12679 of file X86ISelLowering.cpp.
References assert(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), and llvm::MVT::is128BitVector().
Referenced by combineX86ShuffleChain(), lowerShuffleAsInsertPS(), and matchBinaryPermuteShuffle().
|
static |
Definition at line 11797 of file X86ISelLowering.cpp.
References assert(), llvm::sampleprof::Base, llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::Hi, Idx, isSequentialOrUndefInRange(), isUndefInRange(), isUndefUpperHalf(), and Size.
Referenced by combineX86ShuffleChain(), and lowerShuffleWithSSE4A().
|
static |
Try to lower a vector shuffle as a bit shift (shifts in zeros).
Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function matches elements from one of the input vectors shuffled to the left or right with zeroable elements 'shifted in'. It handles both the strictly bit-wise element shifts and the byte shift across an entire 128-bit double quad word lane.
PSHL : (little-endian) left bit shift. [ zz, 0, zz, 2 ] [ -1, 4, zz, -1 ] PSRL : (little-endian) right bit shift. [ 1, zz, 3, zz] [ -1, -1, 7, zz] PSLLDQ : (little-endian) left byte shift [ zz, 0, 1, 2, 3, 4, 5, 6] [ zz, zz, -1, -1, 2, 3, 4, -1] [ zz, zz, zz, zz, zz, zz, -1, 1] PSRLDQ : (little-endian) right byte shift [ 5, 6, 7, zz, zz, zz, zz, zz] [ -1, 5, 6, 7, zz, zz, zz, zz] [ 1, 2, -1, -1, -1, -1, zz, zz]
Definition at line 11640 of file X86ISelLowering.cpp.
References llvm::MVT::getIntegerVT(), llvm::MVT::getVectorVT(), isSequentialOrUndefInRange(), llvm::Left, llvm::Low, Size, llvm::X86ISD::VSHLDQ, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRLDQ, and llvm::X86ISD::VSRLI.
Referenced by lowerShuffleAsShift(), and matchUnaryPermuteShuffle().
|
static |
Definition at line 10059 of file X86ISelLowering.cpp.
References llvm::APInt::extractBits(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorVT(), llvm::MVT::is512BitVector(), llvm::APInt::isAllOnes(), and isSequentialOrUndefInRange().
Referenced by combineX86ShuffleChain().
|
static |
Definition at line 10362 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::ComputeNumSignBits(), createPackShuffleMask(), llvm::APInt::getHighBitsSet(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getScalarValueSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasSSE41(), llvm::isAllOnesOrAllOnesSplat(), llvm::isNullOrNullSplat(), isTargetShuffleEquivalent(), llvm::SDValue::isUndef(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, and llvm::peekThroughBitcasts().
Referenced by lowerShuffleWithPACK(), and matchBinaryShuffle().
|
static |
Definition at line 15634 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), isUndefOrZeroOrInRange(), llvm::SM_SentinelUndef, and std::swap().
Referenced by lowerShuffleWithSHUFPD(), and matchBinaryPermuteShuffle().
|
static |
Definition at line 9917 of file X86ISelLowering.cpp.
References assert(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::createUnpackShuffleMask(), DL, llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), getZeroVector(), llvm::X86Subtarget::hasSSE41(), isSequentialOrUndefOrZeroInRange(), isTargetShuffleEquivalent(), isUndefOrZero(), llvm::M1(), llvm::SM_SentinelUndef, std::swap(), llvm::X86ISD::UNPCKH, and llvm::X86ISD::UNPCKL.
Referenced by matchBinaryShuffle().
|
static |
Helper to determine if In
truncated to DstVT
has the necessary signbits / leading zero bits to be truncated with PACKSS / PACKUS, possibly by converting a SRL node to SRA for sign extension.
Definition at line 20442 of file X86ISelLowering.cpp.
References assert(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::KnownBits::countMinLeadingZeros(), DL, llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getValidShiftAmount(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::X86Subtarget::hasSSSE3(), isFreeToSplitVector(), llvm::Log2_32(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::ISD::SRA, and llvm::ISD::SRL.
Referenced by LowerTruncateVecPackWithSignBits(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Definition at line 38006 of file X86ISelLowering.cpp.
References assert(), llvm::SmallVectorTemplateCommon< T, typename >::data(), llvm::MVT::getIntegerVT(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), getV4X86ShuffleImm(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), is128BitLaneCrossingShuffleMask(), is128BitLaneRepeatedShuffleMask(), llvm::MVT::is128BitVector(), is256BitLaneRepeatedShuffleMask(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isAnyZero(), isSequentialOrUndefInRange(), isUndefOrInRange(), matchShuffleAsBitRotate(), matchShuffleAsShift(), llvm::narrowShuffleMaskElts(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SM_SentinelUndef, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VROTLI, llvm::X86ISD::VSHLDQ, and llvm::X86ISD::VSRLDQ.
Referenced by combineX86ShuffleChain().
|
static |
Definition at line 37852 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::MVT::changeVectorElementType(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::MVT::getIntegerVT(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getOpcode_EXTEND_VECTOR_INREG(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE3(), llvm::X86Subtarget::hasSSE41(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::MVT::isInteger(), isTargetShuffleEquivalent(), isUndefInRange(), isUndefOrEqual(), isUndefOrEqualInRange(), isUndefOrZero(), isUndefOrZeroInRange(), llvm::X86ISD::MOVDDUP, llvm::X86ISD::MOVSHDUP, llvm::X86ISD::MOVSLDUP, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SIGN_EXTEND, llvm::X86ISD::VZEXT_MOVL, and llvm::ISD::ZERO_EXTEND.
Referenced by combineX86ShuffleChain().
|
static |
Definition at line 22436 of file X86ISelLowering.cpp.
References llvm::all_of(), llvm::ISD::AND, assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), CC, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::APInt::getZero(), llvm::X86Subtarget::hasSSE2(), llvm::isAllOnesConstant(), llvm::isNullConstant(), LHS, LowerVectorAllEqual(), llvm::Match, llvm::SelectionDAG::matchBinOpReduction(), matchScalarReduction(), llvm::ISD::OR, llvm::peekThroughBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::SmallVectorBase< Size_T >::size(), and llvm::ISD::TRUNCATE.
Referenced by combineSetCC().
|
static |
Create a vector constant without a load.
SSE/AVX provide the bare minimum functionality to do this, so it's all zeros, all ones, or some derivation that is cheap to calculate.
Definition at line 8434 of file X86ISelLowering.cpp.
References DL, getOnesVector(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::isBuildVectorAllOnes(), and llvm::ISD::isBuildVectorAllZeros().
|
static |
If we are extracting a subvector of a vector select and the select condition is composed of concatenated vectors, try to narrow the select width.
This is a common pattern for AVX1 integer code because 256-bit selects may be legal, but there is almost no integer math/logic available for 256-bit. This function should only be called with legal types (otherwise, the calls to get simple value types will assert).
Definition at line 56989 of file X86ISelLowering.cpp.
References assert(), DL, extract128BitVector(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is128BitVector(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), isFreeToSplitVector(), llvm_unreachable, and llvm::ISD::VSELECT.
Referenced by combineEXTRACT_SUBVECTOR().
|
static |
Definition at line 37836 of file X86ISelLowering.cpp.
References llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::SelectionDAG::getVTList(), llvm::MemSDNode::isSimple(), and llvm::X86ISD::VZEXT_LOAD.
Referenced by combineCVTP2I_CVTTP2I(), combineCVTPH2PS(), combineTargetShuffle(), and combineX86INT_TO_FP().
|
static |
If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the low half of each source vector and does not set any high half elements in the destination vector, narrow the shuffle to half its original size.
Definition at line 41750 of file X86ISelLowering.cpp.
References getHalfShuffleMask(), llvm::ShuffleVectorSDNode::getMask(), llvm::SDNode::getOperand(), getShuffleHalfVectors(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDNode::getValueType(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::TargetLoweringBase::isTypeLegal(), and isUndefUpperHalf().
Referenced by combineShuffle().
|
static |
If both arms of a vector select are concatenated vectors, split the select, and concatenate the result to eliminate a wide (256-bit) vector instruction: vselect Cond, (concat T0, T1), (concat F0, F1) --> concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1)
Definition at line 45601 of file X86ISelLowering.cpp.
References llvm::X86ISD::BLENDV, Cond, DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::hasOneUse(), llvm::EVT::is256BitVector(), isFreeToSplitVector(), N, SplitOpsAndApply(), and llvm::ISD::VSELECT.
Referenced by combineSelect(), and llvm::InstCombinerImpl::visitShuffleVectorInst().
Definition at line 55138 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BRCOND, CC, llvm::X86ISD::CMOV, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NO, llvm::X86::COND_O, llvm::X86ISD::SETCC, and llvm::X86ISD::SETCC_CARRY.
Referenced by combineCMP().
Definition at line 52806 of file X86ISelLowering.cpp.
References llvm::ISD::FMA, llvm::X86ISD::FMADD_RND, llvm::X86ISD::FMADDSUB, llvm::X86ISD::FMADDSUB_RND, llvm::X86ISD::FMSUB, llvm::X86ISD::FMSUB_RND, llvm::X86ISD::FMSUBADD, llvm::X86ISD::FMSUBADD_RND, llvm::X86ISD::FNMADD, llvm::X86ISD::FNMADD_RND, llvm::X86ISD::FNMSUB, llvm::X86ISD::FNMSUB_RND, llvm_unreachable, llvm::ISD::STRICT_FMA, llvm::X86ISD::STRICT_FMSUB, llvm::X86ISD::STRICT_FNMADD, and llvm::X86ISD::STRICT_FNMSUB.
Definition at line 55173 of file X86ISelLowering.cpp.
References assert(), llvm::X86ISD::BRCOND, CC, llvm::X86ISD::CMOV, llvm::X86::COND_E, llvm::X86::COND_NE, llvm::X86ISD::SETCC, and llvm::X86ISD::SETCC_CARRY.
Referenced by combineCMP().
|
static |
Definition at line 58374 of file X86ISelLowering.cpp.
References llvm::StringSwitch< T, R >::Case(), Cond, llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_INVALID, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm::X86::COND_NO, llvm::X86::COND_NP, llvm::X86::COND_NS, llvm::X86::COND_O, llvm::X86::COND_P, llvm::X86::COND_S, and llvm::StringSwitch< T, R >::Default().
|
static |
sext(add_nsw(x, C)) --> add(sext(x), C_sext) zext(add_nuw(x, C)) --> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode.
This can eliminate extend, add, and shift instructions.
Definition at line 53796 of file X86ISelLowering.cpp.
References llvm::Add, llvm::ISD::ADD, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::willNotOverflowAdd(), and llvm::ISD::ZERO_EXTEND.
Referenced by combineSext(), and combineZext().
|
static |
Definition at line 49142 of file X86ISelLowering.cpp.
References llvm::Depth, DL, llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::ISD::isBitwiseLogicOp(), llvm::TargetLoweringBase::isOperationLegalOrPromote(), llvm::SelectionDAG::MaxRecursionDepth, N, PromoteMaskArithmetic(), llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineSext(), combineSignExtendInReg(), combineZext(), and PromoteMaskArithmetic().
|
static |
Definition at line 49196 of file X86ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::EVT::isVector(), llvm_unreachable, N, PromoteMaskArithmetic(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 19299 of file X86ISelLowering.cpp.
References llvm::MVT::changeVectorElementType(), llvm::ISD::FP_ROUND, llvm::SelectionDAG::getEntryNode(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::isVector(), and llvm::ISD::STRICT_FP_ROUND.
|
static |
CMOV of constants requires materializing constant operands in registers.
Try to fold those constants into an 'add' instruction to reduce instruction count. We do this with CMOV rather the generic 'select' because there are earlier folds that may be used to turn select-of-constants into logic hacks.
Definition at line 55762 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::all_of(), llvm::X86ISD::CMOV, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), llvm::X86::mayFoldLoad(), N, std::swap(), X, and Y.
Referenced by combineAdd().
|
static |
Definition at line 54696 of file X86ISelLowering.cpp.
References llvm::sampleprof::Base, DL, llvm::SelectionDAG::getMaskedGather(), and llvm::SelectionDAG::getMaskedScatter().
Referenced by combineGatherScatter().
|
static |
When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP.
Here's the math: RegNodeBase = EntryEBP - RegNodeSize ParentFP = RegNodeBase - ParentFrameOffset Subtracting RegNodeSize takes us to the offset of the registration node, and subtracting the offset (negative on x86) takes us back to the parent FP.
Definition at line 25439 of file X86ISelLowering.cpp.
References llvm::ISD::ADD, llvm::GlobalValue::dropLLVMManglingEscape(), llvm::SelectionDAG::getConstant(), llvm::MachineFunction::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getMCSymbol(), llvm::Value::getName(), llvm::SelectionDAG::getNode(), llvm::MCContext::getOrCreateParentFrameOffsetSymbol(), llvm::TargetLoweringBase::getPointerTy(), getSEHRegistrationNodeSize(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Function::hasPersonalityFn(), llvm::ISD::LOCAL_RECOVER, and llvm::ISD::SUB.
|
static |
If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert.
Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.
Definition at line 51422 of file X86ISelLowering.cpp.
References Addr, assert(), llvm::EVT::changeVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_VECTOR_ELT, ML, and llvm::Offset.
Referenced by combineMaskedLoad().
|
static |
If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store.
Note: It is expected that the degenerate cases of an all-zeros or all-ones mask have already been optimized in IR, so we don't bother with those here.
Definition at line 51558 of file X86ISelLowering.cpp.
References Addr, llvm::EVT::changeVectorElementType(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), getParamsForOneTrueMaskedElt(), llvm::MemSDNode::getPointerInfo(), llvm::SelectionDAG::getStore(), llvm::MaskedStoreSDNode::getValue(), llvm::EVT::getVectorElementType(), llvm::MachinePointerInfo::getWithOffset(), and llvm::Offset.
Referenced by combineMaskedStore().
|
static |
When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code.
Two typical patterns are handled: Pattern1: %2 = sext/zext <N x i8> %1 to <N x i32> %4 = sext/zext <N x i8> %3 to <N x i32> %5 = mul <N x i32> %2, %4
Pattern2: %2 = zext/sext <N x i16> %1 to <N x i32> %4 = zext/sext <N x i16> %3 to <N x i32> or %4 = build_vector <N x i32> C1, ..., CN (C1..CN are constants) %5 = mul <N x i32> %2, %4
There are four mul shrinking modes: If %2 == sext32(trunc8(%2)), i.e., the scalar value range of %2 is -128 to 128, and the scalar value range of %4 is also -128 to 128, generate pmullw+sext32 for it (MULS8 mode). If %2 == zext32(trunc8(%2)), i.e., the scalar value range of %2 is 0 to 255, and the scalar value range of %4 is also 0 to 255, generate pmullw+zext32 for it (MULU8 mode). If %2 == sext32(trunc16(%2)), i.e., the scalar value range of %2 is -32768 to 32767, and the scalar value range of %4 is also -32768 to 32767, generate pmullw+pmulhw for it (MULS16 mode). If %2 == zext32(trunc16(%2)), i.e., the scalar value range of %2 is 0 to 65535, and the scalar value range of %4 is also 0 to 65535, generate pmullw+pmulhuw for it (MULU16 mode).
Definition at line 47591 of file X86ISelLowering.cpp.
References canReduceVMulWidth(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::Function::hasMinSize(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, N, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by combineMul().
|
static |
Definition at line 5733 of file X86ISelLowering.cpp.
References assert(), llvm::APInt::getBitWidth(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineX86ShufflesRecursively(), and getTargetShuffleInputs().
|
static |
Removes unused/repeated shuffle source inputs and adjusts the shuffle mask.
Definition at line 6277 of file X86ISelLowering.cpp.
References isUndef(), llvm::none_of(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SmallVectorBase< Size_T >::size(), and llvm::SM_SentinelUndef.
Referenced by combineHorizOpWithShuffle(), combineX86ShuffleChainWithExtract(), combineX86ShufflesRecursively(), and isHorizontalBinOp().
|
static |
Definition at line 5750 of file X86ISelLowering.cpp.
References llvm::APInt::getZero(), llvm::APInt::setBit(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.
Referenced by combineX86ShuffleChain(), and getTargetShuffleInputs().
|
static |
Extracting a scalar FP value from vector element 0 is free, so extract each operand first, then perform the math as a scalar op.
Definition at line 44925 of file X86ISelLowering.cpp.
References assert(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FABS, llvm::ISD::FADD, llvm::ISD::FCEIL, llvm::ISD::FCOPYSIGN, llvm::ISD::FDIV, llvm::ISD::FFLOOR, llvm::ISD::FMA, llvm::ISD::FMAD, llvm::X86ISD::FMAX, llvm::ISD::FMAXIMUM, llvm::ISD::FMAXNUM, llvm::ISD::FMAXNUM_IEEE, llvm::X86ISD::FMIN, llvm::ISD::FMINIMUM, llvm::ISD::FMINNUM, llvm::ISD::FMINNUM_IEEE, llvm::ISD::FMUL, llvm::ISD::FNEARBYINT, llvm::X86ISD::FRCP, llvm::ISD::FREM, llvm::ISD::FRINT, llvm::ISD::FROUND, llvm::ISD::FROUNDEVEN, llvm::X86ISD::FRSQRT, llvm::ISD::FSQRT, llvm::ISD::FSUB, llvm::ISD::FTRUNC, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), llvm_unreachable, llvm::SDNode::ops(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SELECT, llvm::ISD::SETCC, and llvm::ISD::VSELECT.
Referenced by combineExtractVectorElt().
|
static |
Scalarize a vector store, bitcasting to TargetVT to determine the scalar type.
Definition at line 24634 of file X86ISelLowering.cpp.
References assert(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getStore(), llvm::MVT::getStoreSize(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::EVT::is128BitVector(), llvm::MVT::is128BitVector(), llvm::Offset, Ptr, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::TokenFactor.
Referenced by combineStore(), and llvm::TargetLowering::expandUnalignedStore().
|
static |
Definition at line 3745 of file X86ISelLowering.cpp.
References assert(), canWidenShuffleElements(), llvm::narrowShuffleMaskElts(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by canonicalizeShuffleMaskWithHorizOp(), canScaleShuffleElements(), combineBlendOfPermutes(), combineEXTRACT_SUBVECTOR(), combineHorizOpWithShuffle(), combineTargetShuffle(), combineX86ShuffleChain(), and isHorizontalBinOp().
|
static |
Definition at line 30753 of file X86ISelLowering.cpp.
References llvm::AtomicRMWInst::Add, llvm::AtomicRMWInst::And, llvm::User::getOperand(), llvm::AtomicRMWInst::getOperation(), llvm::Value::hasOneUse(), I, llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLT, llvm::PatternMatch::m_AllOnes(), llvm::PatternMatch::m_c_Add(), llvm::PatternMatch::m_c_And(), llvm::PatternMatch::m_c_ICmp(), llvm::PatternMatch::m_c_Or(), llvm::PatternMatch::m_c_Xor(), llvm::PatternMatch::m_ICmp(), llvm::PatternMatch::m_OneUse(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificICmp(), llvm::PatternMatch::m_Sub(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_ZeroInt(), llvm::PatternMatch::match(), llvm::AtomicRMWInst::Or, llvm::AtomicRMWInst::Sub, llvm::Instruction::user_back(), and llvm::AtomicRMWInst::Xor.
|
static |
Horizontal vector math instructions may be slower than normal math with shuffles.
Limit horizontal op codegen based on size/speed trade-offs, uarch implementation, and likely shuffle complexity of the alternate sequence.
Definition at line 19480 of file X86ISelLowering.cpp.
References llvm::SelectionDAG::shouldOptForSize().
Referenced by canonicalizeShuffleMaskWithHorizOp(), combineArithReduction(), combineVectorHADDSUB(), isHorizontalBinOp(), lowerAddSubToHorizontalOp(), and LowerUINT_TO_FP_i64().
|
static |
Definition at line 43477 of file X86ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::BUILD_VECTOR, DL, llvm::ISD::FREEZE, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm_unreachable, llvm::ISD::OR, llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, signExtendBitcastSrcVector(), llvm::ISD::TRUNCATE, llvm::ISD::VSELECT, and llvm::ISD::XOR.
Referenced by combineBitcastvxi1(), and signExtendBitcastSrcVector().
|
static |
Definition at line 20212 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::Hi, llvm::Lo, and llvm::ISD::TRUNCATE.
Referenced by LowerSIGN_EXTEND_Mask(), and LowerZERO_EXTEND_Mask().
|
static |
Generic routine to split vector shuffle into half-sized shuffles.
This routine just extracts two subvectors, shuffles them independently, and then concatenates them back together. This should work effectively with all AVX vector shuffle types.
Definition at line 14539 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::Hi, llvm::Lo, llvm::peekThroughBitcasts(), and splitVector().
Referenced by lower256BitShuffle(), lower512BitShuffle(), lowerShuffleAsLanePermuteAndShuffle(), lowerShuffleAsSplitOrBlend(), lowerV64I8Shuffle(), and lowerV8F32Shuffle().
|
static |
Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back.
Definition at line 23109 of file X86ISelLowering.cpp.
References assert(), CC, llvm::ISD::CONCAT_VECTORS, Cond, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::EVT::isInteger(), LHS, RHS, llvm::ISD::SETCC, and splitVector().
Referenced by LowerVSETCC().
SDValue SplitOpsAndApply | ( | SelectionDAG & | DAG, |
const X86Subtarget & | Subtarget, | ||
const SDLoc & | DL, | ||
EVT | VT, | ||
ArrayRef< SDValue > | Ops, | ||
F | Builder, | ||
bool | CheckBWI = true |
||
) |
Definition at line 4219 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, extractSubVector(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasSSE2(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::X86Subtarget::useAVX512Regs(), and llvm::X86Subtarget::useBWIRegs().
Referenced by combineArithReduction(), combineMulToPMADDWD(), combineMulToPMULDQ(), combineToHorizontalAddSub(), createPSADBW(), createVPDPBUSD(), detectPMADDUBSW(), matchPMADDWD(), matchPMADDWD_2(), and narrowVectorSelect().
|
static |
Definition at line 4141 of file X86ISelLowering.cpp.
References assert(), extractSubVector(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::SelectionDAG::isSplatValue(), and llvm::Lo.
Referenced by combineAndShuffleNot(), combineArithReduction(), combineMinMaxReduction(), LowerMULO(), splitAndLowerShuffle(), splitIntVSETCC(), splitVectorOp(), splitVectorStore(), and truncateVectorWithPACK().
|
static |
Break a binary integer operation into 2 half sized ops and then concatenate the result back.
Definition at line 4201 of file X86ISelLowering.cpp.
References assert(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), and splitVectorOp().
Referenced by combineEXTRACT_SUBVECTOR(), LowerABD(), LowerADDSAT_SUBSAT(), lowerAddSub(), LowerAVG(), LowerMINMAX(), LowerMUL(), LowerMULH(), LowerRotate(), and LowerShift().
|
static |
Break an unary integer operation into 2 half sized ops and then concatenate the result back.
Definition at line 4185 of file X86ISelLowering.cpp.
References assert(), llvm::EVT::getVectorNumElements(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), and splitVectorOp().
Referenced by LowerABS(), LowerAVXExtend(), LowerBITREVERSE(), LowerBITREVERSE_XOP(), LowerSIGN_EXTEND(), LowerVectorCTLZ(), LowerVectorCTLZ_AVX512CDI(), and LowerVectorCTPOP().
|
static |
Break an operation into 2 half sized ops and then concatenate the results.
Definition at line 4160 of file X86ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::GetSplitDestVTs(), I, and splitVector().
Referenced by LowerFunnelShift(), splitVectorIntBinary(), and splitVectorIntUnary().
|
static |
Change a vector store into a pair of half-size vector stores.
Definition at line 24600 of file X86ISelLowering.cpp.
References assert(), DL, llvm::TypeSize::getFixed(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStore(), llvm::EVT::getStoreSize(), llvm::SDValue::getValueType(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), splitVector(), and llvm::ISD::TokenFactor.
Referenced by combineStore(), and LowerStore().
|
static |
Definition at line 29151 of file X86ISelLowering.cpp.
References supportedVectorShiftWithImm().
Referenced by LowerFunnelShift(), and LowerShiftByScalarVariable().
|
static |
Definition at line 29122 of file X86ISelLowering.cpp.
References assert(), llvm::EVT::getScalarSizeInBits(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::EVT::isSimple(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and llvm::X86Subtarget::useAVX512Regs().
Referenced by combineAndMaskToShift(), LowerFunnelShift(), LowerRotate(), LowerShiftByScalarImmediate(), and supportedVectorShiftWithBaseAmnt().
|
static |
Definition at line 29158 of file X86ISelLowering.cpp.
References assert(), llvm::EVT::getScalarSizeInBits(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::EVT::isSimple(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and llvm::X86Subtarget::useAVX512Regs().
Referenced by combineSelect(), combineShiftLeft(), combineShiftRightArithmetic(), combineShiftRightLogical(), LowerFunnelShift(), LowerRotate(), and LowerShift().
|
static |
Definition at line 2820 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, and llvm::ISD::SETULT.
Referenced by TranslateX86CC().
|
static |
Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make.
Definition at line 2841 of file X86ISelLowering.cpp.
References llvm::X86::COND_A, llvm::X86::COND_AE, llvm::X86::COND_B, llvm::X86::COND_BE, llvm::X86::COND_E, llvm::X86::COND_INVALID, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm::X86::COND_NP, llvm::X86::COND_NS, llvm::X86::COND_P, llvm::X86::COND_S, DL, llvm::SelectionDAG::getConstant(), llvm::ISD::isNON_EXTLoad(), LHS, llvm_unreachable, RHS, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, std::swap(), and TranslateIntegerX86CC().
|
static |
Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs.
Definition at line 23046 of file X86ISelLowering.cpp.
References llvm_unreachable, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, and std::swap().
Referenced by combineConcatVectorOps(), and LowerVSETCC().
|
static |
If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just pre-promote its result type since vXi1 vectors don't get promoted during type legalization.
Definition at line 54242 of file X86ISelLowering.cpp.
References CC, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getVectorElementType(), llvm::X86Subtarget::hasAVX512(), llvm::EVT::isVector(), LHS, RHS, and llvm::ISD::TRUNCATE.
Referenced by combineSetCC().
|
static |
Helper to recursively truncate vector elements in half with PACKSS/PACKUS.
It makes use of the fact that vectors with enough leading sign/zero bits prevent the PACKSS/PACKUS from saturating the results. AVX2 (Int256) sub-targets require extra shuffling as the PACK*S operates within each 128-bit lane.
Definition at line 20300 of file X86ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, extractSubVector(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::Hi, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::EVT::is512BitVector(), llvm::isPowerOf2_32(), llvm::EVT::isVector(), LHS, llvm::Lo, llvm::narrowShuffleMaskElts(), llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, RHS, splitVector(), truncateVectorWithPACK(), and widenSubVector().
Referenced by combineTruncateWithSat(), LowerTruncateVecPackWithSignBits(), llvm::X86TargetLowering::ReplaceNodeResults(), truncateVectorWithPACK(), truncateVectorWithPACKSS(), and truncateVectorWithPACKUS().
|
static |
Truncate using inreg sign extension and X86ISD::PACKSS.
Definition at line 20430 of file X86ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getValueType(), llvm::X86ISD::PACKSS, llvm::ISD::SIGN_EXTEND_INREG, and truncateVectorWithPACK().
Referenced by LowerTruncateVecPack().
|
static |
Truncate using inreg zero extension (AND mask) and X86ISD::PACKUS.
e.g. trunc <8 x i32> X to <8 x i16> --> MaskX = X & 0xffff (clear high bits to prevent saturation) packus (extract_subv MaskX, 0), (extract_subv MaskX, 1)
Definition at line 20422 of file X86ISelLowering.cpp.
References DL, llvm::SelectionDAG::getZeroExtendInReg(), llvm::X86ISD::PACKUS, and truncateVectorWithPACK().
Referenced by LowerTruncateVecPack().
|
static |
Definition at line 58870 of file X86ISelLowering.cpp.
Referenced by llvm::X86TargetLowering::getRegForInlineAsmConstraint().
|
static |
Definition at line 19096 of file X86ISelLowering.cpp.
References llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasSSE2(), llvm::ISD::SINT_TO_FP, and llvm::ISD::UINT_TO_FP.
Referenced by vectorizeExtractedCast().
|
static |
Definition at line 2940 of file X86ISelLowering.cpp.
References llvm::X86Subtarget::canExtendTo512DQ(), and llvm::MVT::is512BitVector().
Referenced by canonicalizeBitSelect(), LowerADDSAT_SUBSAT(), LowerFunnelShift(), and LowerRotate().
|
static |
Given a scalar cast operation that is extracted from a vector, try to vectorize the cast op followed by extraction.
This will avoid an expensive round-trip between XMM and GPR.
Definition at line 19121 of file X86ISelLowering.cpp.
References DL, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::isNullConstant(), and useVectorCast().
|
static |
Widen a mask vector to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.
Definition at line 4041 of file X86ISelLowering.cpp.
References llvm::SDValue::getSimpleValueType(), widenMaskVectorType(), and widenSubVector().
Referenced by combineCMP(), ExtractBitFromMaskVector(), lower1BitShuffle(), lower1BitShuffleAsKSHIFTR(), and LowerEXTRACT_SUBVECTOR().
|
static |
Widen a mask vector type to a minimum of v8i1/v16i1 to allow use of KSHIFT and bitcast with integer types.
Definition at line 4031 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getVectorElementType(), and llvm::MVT::getVectorNumElements().
Referenced by insert1BitVector(), LowerCONCAT_VECTORSvXi1(), and widenMaskVector().
|
static |
Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.
Definition at line 4003 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getFixedSizeInBits(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), getZeroVector(), and llvm::ISD::INSERT_SUBVECTOR.
Referenced by canonicalizeShuffleMaskWithHorizOp(), combineBitcastvxi1(), combineConcatVectorOps(), combineTargetShuffle(), combineX86ShuffleChain(), combineX86ShufflesRecursively(), createVariablePermute(), getAVX512Node(), getAVX512TruncNode(), LowerCONCAT_VECTORSvXi1(), lowerShuffleWithPERMV(), LowerTruncateVecPack(), LowerTruncateVecPackWithSignBits(), llvm::X86TargetLowering::ReplaceNodeResults(), llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(), truncateVectorWithPACK(), widenMaskVector(), and widenSubVector().
|
static |
Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef.
Definition at line 4017 of file X86ISelLowering.cpp.
References assert(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::getValueSizeInBits(), llvm::MVT::getVectorVT(), and widenSubVector().
|
static |
Referenced by llvm::X86TargetLowering::getJumpConditionMergingParams().
|
static |
Referenced by llvm::X86TargetLowering::getJumpConditionMergingParams().
|
static |
Referenced by llvm::X86TargetLowering::getJumpConditionMergingParams().
|
static |
Referenced by llvm::X86TargetLowering::getJumpConditionMergingParams().
|
static |
Referenced by llvm::X86TargetLowering::getPrefLoopAlignment().
Definition at line 27801 of file X86ISelLowering.cpp.
const unsigned FPStateSizeInBits = FPStateSize * 8 |
Definition at line 27802 of file X86ISelLowering.cpp.
|
static |
Referenced by combineMul().
Definition at line 27800 of file X86ISelLowering.cpp.
Referenced by createSetFPEnvNodes().