LLVM 22.0.0git
|
#include "AArch64ISelLowering.h"
#include "AArch64CallingConvention.h"
#include "AArch64ExpandImm.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64PerfectShuffle.h"
#include "AArch64RegisterInfo.h"
#include "AArch64Subtarget.h"
#include "AArch64TargetMachine.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "Utils/AArch64BaseInfo.h"
#include "Utils/AArch64SMEAttributes.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ObjCARCUtil.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/SipHash.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/TargetParser/Triple.h"
#include <algorithm>
#include <bitset>
#include <cassert>
#include <cctype>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <optional>
#include <tuple>
#include <utility>
#include <vector>
#include "AArch64GenAsmMatcher.inc"
Go to the source code of this file.
Classes | |
struct | GenericSetCCInfo |
Helper structure to keep track of ISD::SET_CC operands. More... | |
struct | AArch64SetCCInfo |
Helper structure to keep track of a SET_CC lowered into AArch64 code. More... | |
union | SetCCInfo |
Helper structure to keep track of SetCC information. More... | |
struct | SetCCInfoAndKind |
Helper structure to be able to read SetCC information. More... |
Macros | |
#define | DEBUG_TYPE "aarch64-lower" |
#define | GET_REGISTER_MATCHER |
Enumerations | |
enum class | PredicateConstraint { Uph , Upl , Upa } |
enum class | ReducedGprConstraint { Uci , Ucj } |
Variables | |
cl::opt< bool > | EnableAArch64ELFLocalDynamicTLSGeneration ("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false)) |
static cl::opt< bool > | EnableOptimizeLogicalImm ("aarch64-enable-logical-imm", cl::Hidden, cl::desc("Enable AArch64 logical imm instruction " "optimization"), cl::init(true)) |
static cl::opt< bool > | EnableCombineMGatherIntrinsics ("aarch64-enable-mgather-combine", cl::Hidden, cl::desc("Combine extends of AArch64 masked " "gather intrinsics"), cl::init(true)) |
static cl::opt< bool > | EnableExtToTBL ("aarch64-enable-ext-to-tbl", cl::Hidden, cl::desc("Combine ext and trunc to TBL"), cl::init(true)) |
static cl::opt< unsigned > | MaxXors ("aarch64-max-xors", cl::init(16), cl::Hidden, cl::desc("Maximum of xors")) |
cl::opt< bool > | EnableSVEGISel ("aarch64-enable-gisel-sve", cl::Hidden, cl::desc("Enable / disable SVE scalable vectors in Global ISel"), cl::init(false)) |
static cl::opt< bool > | UseFEATCPACodegen ("aarch64-use-featcpa-codegen", cl::Hidden, cl::desc("Generate ISD::PTRADD nodes for pointer arithmetic in " "SelectionDAG for FEAT_CPA"), cl::init(false)) |
constexpr MVT | CondCodeVT = MVT::i32 |
Value type used for condition codes. | |
constexpr MVT | FlagsVT = MVT::i32 |
Value type used for NZCV flags. | |
static const MCPhysReg | GPRArgRegs [] |
static const MCPhysReg | FPRArgRegs [] |
true | |
Simplify Addr given that the top byte of it is ignored by HW during / address translation. |
#define DEBUG_TYPE "aarch64-lower" |
Definition at line 109 of file AArch64ISelLowering.cpp.
#define GET_REGISTER_MATCHER |
Definition at line 12535 of file AArch64ISelLowering.cpp.
|
strong |
Enumerator | |
---|---|
Uph | |
Upl | |
Upa |
Definition at line 12825 of file AArch64ISelLowering.cpp.
|
strong |
Enumerator | |
---|---|
Uci | |
Ucj |
Definition at line 12889 of file AArch64ISelLowering.cpp.
|
static |
Definition at line 8944 of file AArch64ISelLowering.cpp.
References llvm::CCState::AllocateStack(), llvm::TargetLowering::CallLoweringInfo::Args, llvm::CallingConv::ARM64EC_Thunk_X64, assert(), llvm::TargetLowering::CallLoweringInfo::CallConv, llvm::AArch64TargetLowering::CCAssignFnForCall(), llvm::TargetLowering::CallLoweringInfo::DAG, llvm::CCValAssign::Full, llvm::SelectionDAG::getDataLayout(), llvm::EVT::getSimpleVT(), llvm::TargetLoweringBase::getValueType(), llvm::AArch64Subtarget::isCallingConvWin64(), llvm::EVT::isSimple(), llvm::TargetLowering::CallLoweringInfo::IsVarArg, llvm::ISD::ArgFlagsTy::isVarArg(), llvm::TargetLowering::CallLoweringInfo::Outs, and llvm::SmallVectorTemplateCommon< T, typename >::size().
|
static |
Definition at line 22049 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::all_of(), llvm::ISD::ANY_EXTEND, areLoadedOffsetButOtherwiseSame(), llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), llvm::get(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), isLoadOrMultipleLoads(), llvm::ISD::SIGN_EXTEND, Size, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::SUB, llvm::ISD::ZERO_EXTEND, and llvm::zip().
Referenced by areLoadedOffsetButOtherwiseSame(), and performExtBinopLoadFold().
Calculates what the pre-extend type is, based on the extension operation node provided by Extend
.
In the case that Extend
is a SIGN_EXTEND or a ZERO_EXTEND, the pre-extend type is pulled directly from the operand, while other extend operations need a bit more inspection to get this information.
Extend | The SDNode from the DAG that represents the extend operation |
Extend
source type, or MVT::Other
if no valid type can be determined Definition at line 19293 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::VTSDNode::getVT(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, and llvm::ISD::ZERO_EXTEND.
Referenced by performBuildShuffleExtendCombine().
|
static |
Return true if the call convention supports varargs Currently only those that pass varargs like the C calling convention does are eligible Calling conventions listed in this function must also be properly handled in AArch64Subtarget::isCallingConvWin64.
Definition at line 8930 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::AArch64_SVE_VectorCall, llvm::CallingConv::C, and llvm::CallingConv::PreserveNone.
|
static |
Returns true if Val
is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.
See CMP;CCMP matching.
CanNegate | Set to true if we can negate the whole sub-tree just by changing the conditions on the SETCC tests. (this means we can call emitConjunctionRec() with Negate==true on this sub-tree) |
MustBeFirst | Set to true if this subtree needs to be negated and we cannot do the negation naturally. We are required to emit the subtree first in this case. |
WillNegate | Is true if are called when the result of this subexpression must be negated. This happens when the outer expression is an OR. We can use this fact to know that we have a double negation (or (or ...) ...) that can be implemented for free. |
Definition at line 3867 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), canEmitConjunction(), llvm::Depth, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::ISD::OR, and llvm::ISD::SETCC.
Referenced by canEmitConjunction(), canEmitConjunction(), emitConjunction(), and emitConjunctionRec().
|
static |
Return true if the calling convention is one that we can guarantee TCO for.
Definition at line 8902 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::Fast, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.
Referenced by llvm::AArch64CallLowering::isEligibleForTailCallOptimization(), mayTailCallThisCC(), mayTailCallThisCC(), shouldGuaranteeTCO(), and shouldGuaranteeTCO().
|
static |
Definition at line 16403 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::Add, assert(), llvm::dyn_cast_or_null(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSplatValue(), llvm::SDValue::getValueType(), llvm::EVT::isScalableVT(), and llvm::ISD::SRL.
Referenced by trySimplifySrlAddToRshrnb().
|
static |
Definition at line 4443 of file AArch64ISelLowering.cpp.
References assert(), Cond, DL, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getResNo(), llvm::AArch64CC::HS, and llvm::AArch64CC::LO.
Referenced by lowerADDSUBO_CARRY().
|
static |
changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
Definition at line 3503 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::LE, llvm_unreachable, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.
Referenced by changeFPCCToANDAArch64CC(), and changeVectorFPCCToAArch64CC().
|
static |
Convert a DAG fp condition code to an AArch64 CC.
This differs from changeFPCCToAArch64CC in that it returns cond codes that should be AND'ed instead of OR'ed.
Definition at line 3566 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, assert(), changeFPCCToAArch64CC(), llvm::AArch64CC::LE, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETONE, llvm::ISD::SETUEQ, and llvm::AArch64CC::VC.
Referenced by emitConjunctionRec().
|
static |
changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC
Definition at line 3474 of file AArch64ISelLowering.cpp.
References RHS.
Referenced by emitConjunctionRec(), and getAArch64Cmp().
|
static |
changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions.
Fewer operations are available without a real NZCV register, so we have to use less efficient combinations to get the same effect.
Definition at line 3596 of file AArch64ISelLowering.cpp.
References changeFPCCToAArch64CC(), llvm::AArch64CC::GE, llvm::AArch64CC::MI, llvm::ISD::SETO, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, and llvm::ISD::SETUO.
Referenced by emitFloatCompareMask().
|
static |
Definition at line 25263 of file AArch64ISelLowering.cpp.
References llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::cast(), llvm::ISD::Constant, llvm::LoadSDNode::getExtensionType(), llvm::MemSDNode::getMemoryVT(), llvm::VTSDNode::getVT(), llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::TargetConstant, and llvm::ISD::ZEXTLOAD.
Referenced by performCONDCombine(), and performSETCCCombine().
|
static |
Definition at line 9210 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::EVT::getSizeInBits(), and llvm::SDValue::getValueType().
|
static |
Definition at line 22397 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), N, and Opc.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 22662 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm_unreachable, and N.
Referenced by performIntrinsicCombine().
|
static |
Combines a node carrying the intrinsic aarch64_sve_prf<T>_gather_scalar_offset into a node that uses aarch64_sve_prfb_gather_uxtw_index when the scalar offset passed to aarch64_sve_prf<T>_gather_scalar_offset is not a valid immediate for the sve gather prefetch instruction with vector plus immediate addressing mode.
Definition at line 27156 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), isValidImmForSVEVecImmAddrMode(), N, SDValue(), and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 22587 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), N, and Opc.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 22568 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), N, and Opc.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 22604 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_VECTOR_ELT, N, and Opc.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 15358 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), DL, llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), resolveBuildVector(), SDValue(), tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), tryAdvSIMDModImm321s(), tryAdvSIMDModImm64(), tryAdvSIMDModImm8(), tryAdvSIMDModImmFP(), trySVESplat64(), and llvm::APInt::zext().
|
static |
Definition at line 14331 of file AArch64ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::EVT::is128BitVector(), and WidenVector().
|
static |
Definition at line 29804 of file AArch64ISelLowering.cpp.
References llvm::cast(), convertToScalableVector(), DL, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getNode(), getPredicateForFixedLengthVector(), llvm::SDValue::getValueType(), llvm::isBitwiseNot(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::SETCC, and llvm::ISD::SETNE.
Referenced by LowerVectorMatch().
|
static |
Definition at line 29754 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and llvm::EVT::isFixedLengthVector().
Referenced by convertLocVTToValVT(), GenerateFixedLengthSVETBL(), getVCIXISDNodeWCHAIN(), getWideningInterleave(), llvm::RISCVTargetLowering::joinRegisterPartsIntoValue(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaVID(), lowerFixedVectorSegLoadIntrinsics(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), LowerVectorMatch(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), lowerVZIP(), matchSplatAsGather(), performFP_TO_INTCombine(), performSVEMulAddSubCombine(), and trySVESplat64().
|
static |
Definition at line 22641 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, isAllActivePredicate(), N, Opc, and SDValue().
Referenced by performIntrinsicCombine().
|
static |
Definition at line 29743 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::ISD::INSERT_SUBVECTOR, and llvm::EVT::isScalableVector().
Referenced by combineToVCPOP(), convertFixedMaskToScalableVector(), convertValVTToLocVT(), GenerateFixedLengthSVETBL(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerCttzElts(), lowerFixedVectorSegLoadIntrinsics(), lowerFixedVectorSegStoreIntrinsics(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerScalarInsert(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), LowerVectorMatch(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), lowerVZIP(), matchSplatAsGather(), llvm::RISCVTargetLowering::PerformDAGCombine(), performFP_TO_INTCombine(), performSVEMulAddSubCombine(), processVCIXOperands(), llvm::RISCVTargetLowering::ReplaceNodeResults(), and llvm::RISCVTargetLowering::splitValueIntoRegisterParts().
|
static |
Definition at line 28400 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getTargetConstant(), llvm::DataLayout::isBigEndian(), SDValue(), llvm::SelectionDAG::SplitScalar(), and std::swap().
Referenced by ReplaceCMP_SWAP_128Results().
Definition at line 17387 of file AArch64ISelLowering.cpp.
References assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), llvm::SmallVectorImpl< T >::clear(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::Instruction::eraseFromParent(), llvm::ConstantVector::get(), llvm::FixedVectorType::get(), llvm::User::getOperand(), llvm::Value::getType(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::Value::replaceAllUsesWith(), Results, and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().
|
static |
Definition at line 17369 of file AArch64ISelLowering.cpp.
References llvm::cast(), createTblShuffleMask(), llvm::PoisonValue::get(), and llvm::VectorType::getElementType().
Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().
|
static |
Definition at line 17347 of file AArch64ISelLowering.cpp.
References llvm::cast(), createTblShuffleMask(), llvm::PoisonValue::get(), and llvm::VectorType::getElementType().
Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().
|
static |
Definition at line 17324 of file AArch64ISelLowering.cpp.
Referenced by createTblShuffleForSExt(), and createTblShuffleForZExt().
|
static |
Definition at line 28204 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVectorIdxConstant(), N, Results, and llvm::ISD::SCALAR_TO_VECTOR.
|
static |
Definition at line 3707 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), DL, FlagsVT, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), isCMN(), llvm::EVT::isFloatingPoint(), llvm::isNullConstant(), LHS, llvm::SelectionDAG::ReplaceAllUsesWith(), RHS, and llvm::ISD::SUB.
Referenced by emitConjunctionRec(), and getAArch64Cmp().
|
static |
can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" which can be implemented as: cmp C ccmp D, inv(CD), CC ccmp A, CA, inv(CD) ccmp B, CB, inv(CA) check for CB flags
A counterexample is "or (and A B) (and C D)" which translates to not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we can only implement 1 of the inner (not) operations, but not both! Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
Definition at line 3811 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::dyn_cast(), FlagsVT, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SelectionDAG::getSubtarget(), isCMN(), llvm::isNullConstant(), LHS, RHS, and llvm::ISD::SUB.
Referenced by emitConjunctionRec().
|
static |
Emit expression as a conjunction (a series of CCMP/CFCMP ops).
In some cases this is even possible with OR operations in the expression. See CMP;CCMP matching.
Definition at line 4041 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, canEmitConjunction(), emitConjunctionRec(), and SDValue().
Referenced by getAArch64Cmp(), LowerBRCOND(), and performANDSETCCCombine().
|
static |
Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
See CMP;CCMP matching. Tries to transform the given i1 producing node Val
to a series compare and conditional compare operations.
OutCC
to the flags that should be tested or returns SDValue() if transformation was not possible. Negate
is true if we want this sub-tree being negated just by changing SETCC conditions. Definition at line 3929 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, llvm::ISD::AND, assert(), canEmitConjunction(), llvm::cast(), changeFPCCToANDAArch64CC(), changeIntCCToAArch64CC(), DL, emitComparison(), emitConditionalComparison(), emitConjunctionRec(), llvm::AArch64CC::getInvertedCondCode(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::hasOneUse(), LHS, llvm::ISD::OR, RHS, llvm::ISD::SETCC, and std::swap().
Referenced by emitConjunction(), and emitConjunctionRec().
|
static |
For SELECT_CC, when the true/false values are (-1, 0) and the compared values are scalars, try to emit a mask generating vector instruction.
Definition at line 11665 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, assert(), changeVectorFPCCToAArch64CC(), DL, llvm::dyn_cast(), emitVectorComparison(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::ISD::getSetCCInverse(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorVT(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ConstantSDNode::isAllOnes(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::ConstantSDNode::isZero(), LHS, llvm::ISD::OR, RHS, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETUNE, and llvm::ISD::SETUO.
|
static |
Definition at line 8113 of file AArch64ISelLowering.cpp.
References DL, llvm::TPIDR2Object::FrameIndex, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getCopyToReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getFrameIndex(), llvm::TargetLoweringBase::getFrameIndexTy(), llvm::TargetLoweringBase::getLibcallCallingConv(), llvm::TargetLoweringBase::getLibcallName(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::AArch64TargetLowering::getPointerTy(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getRegisterMask(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::AArch64FunctionInfo::getTPIDR2Obj(), llvm::SDValue::getValue(), llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, TRI, and llvm::TPIDR2Object::Uses.
|
static |
Definition at line 8090 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getExternalSymbol(), llvm::MachineFunction::getInfo(), llvm::TargetLoweringBase::getLibcallCallingConv(), llvm::TargetLoweringBase::getLibcallName(), llvm::SelectionDAG::getMachineFunction(), llvm::AArch64TargetLowering::getPointerTy(), llvm::PointerType::getUnqual(), llvm::Type::getVoidTy(), Info, llvm::TargetLowering::LowerCallTo(), llvm::TargetLowering::CallLoweringInfo::setChain(), llvm::TargetLowering::CallLoweringInfo::setDebugLoc(), llvm::TargetLowering::CallLoweringInfo::setLibCallee(), and llvm::AArch64FunctionInfo::setSMESaveBufferUsed().
|
static |
Definition at line 3687 of file AArch64ISelLowering.cpp.
References assert(), DL, FlagsVT, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), LHS, RHS, and llvm::ISD::STRICT_FP_EXTEND.
|
static |
Emit vector comparison for floating-point values, producing a mask.
Definition at line 11616 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::AArch64CC::EQ, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::AArch64CC::GE, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorVT(), llvm::AArch64CC::GT, llvm::ISD::INSERT_VECTOR_ELT, llvm::AArch64CC::LE, LHS, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, RHS, and SDValue().
Referenced by emitFloatCompareMask().
|
static |
Definition at line 355 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::SDNode::getConstantOperandVal(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::INTRINSIC_WO_CHAIN, and llvm::isUInt().
|
static |
Definition at line 25014 of file AArch64ISelLowering.cpp.
References llvm::cast(), Changed, llvm::EVT::changeVectorElementType(), llvm::dyn_cast_or_null(), foldIndexIntoBase(), llvm::SDValue::getConstantOperandVal(), llvm::AArch64Subtarget::getMaxSVEVectorSizeInBits(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSplatValue(), llvm::SelectionDAG::getStepVector(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isVectorShrinkable(), N, RHS, llvm::ISD::SHL, llvm::ISD::STEP_VECTOR, llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.
Referenced by performMaskedGatherScatterCombine().
|
static |
Definition at line 21474 of file AArch64ISelLowering.cpp.
References Cond, DL, getCondCode(), llvm::SelectionDAG::getNode(), llvm::isNullConstant(), LHS, llvm::AArch64CC::LO, N, RHS, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 25691 of file AArch64ISelLowering.cpp.
References llvm::cast(), Cond, DL, llvm::AArch64CC::EQ, llvm::ConstantSDNode::getAPIntValue(), getCondCode(), llvm::SDNode::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::isa(), isCMP(), llvm::AArch64CC::NE, SDValue(), std::swap(), X, and Y.
Referenced by performCSELCombine().
|
static |
Definition at line 25643 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::BitWidth, llvm::ISD::CTTZ, llvm::AArch64CC::EQ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::isNullConstant(), N, llvm::AArch64CC::NE, SDValue(), llvm::ISD::TRUNCATE, and X.
Referenced by performCSELCombine().
|
static |
Definition at line 25886 of file AArch64ISelLowering.cpp.
References Default, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), isAllActivePredicate(), llvm::AArch64CC::NE, and SDValue().
Referenced by performCSELCombine().
|
static |
Definition at line 24964 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::Add, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSplatValue(), llvm::EVT::getVectorElementType(), llvm::EVT::isVector(), llvm::ISD::MUL, llvm::Offset, and llvm::ISD::SHL.
Referenced by findMoreOptimalIndexType().
|
static |
Definition at line 21450 of file AArch64ISelLowering.cpp.
References getCSETCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::AArch64CC::HS, isCMP(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::AArch64CC::LO, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X, X, #0.
Definition at line 18720 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllOnes(), llvm::EVT::isVector(), N, SDValue(), and llvm::ISD::SETGE.
Referenced by combineXor(), and performXorCombine().
|
static |
Definition at line 30800 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ArrayRef(), assert(), llvm::EVT::changeTypeToInteger(), convertFromScalableVector(), convertToScalableVector(), llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getContext(), llvm::AArch64Subtarget::getMaxSVEVectorSizeInBits(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVScale(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::AArch64Subtarget::isNeonAvailable(), llvm::ShuffleVectorInst::isSingleSourceMask(), llvm::maxUIntN(), llvm::ISD::MUL, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::ISD::SPLAT_VECTOR, and llvm::AArch64::SVEBitsPerBlock.
|
static |
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations to build the shuffle.
ID is the perfect-shuffle
Definition at line 14083 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, GeneratePerfectShuffle(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getExtFactor(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::ISD::INSERT_VECTOR_ELT, LHS, llvm_unreachable, OP_COPY, OP_VDUP0, OP_VDUP1, OP_VDUP2, OP_VDUP3, OP_VEXT1, OP_VEXT2, OP_VEXT3, OP_VREV, OP_VTRNL, OP_VTRNR, OP_VUZPL, OP_VUZPR, OP_VZIPL, OP_VZIPR, llvm::PerfectShuffleTable, RHS, and WidenVector().
Referenced by GeneratePerfectShuffle(), GeneratePerfectShuffle(), GeneratePerfectShuffle(), and LowerVECTOR_SHUFFLE().
|
static |
Definition at line 14244 of file AArch64ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::SDValue::isUndef(), isZerosVector(), llvm::Offset, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and std::swap().
|
static |
Definition at line 4110 of file AArch64ISelLowering.cpp.
References assert(), llvm::CallingConv::C, llvm::cast(), changeIntCCToAArch64CC(), DL, llvm::dyn_cast(), emitComparison(), emitConjunction(), getCmpOperandFoldingProfit(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::ISD::getSetCCSwappedOperands(), llvm::SelectionDAG::getSignedConstant(), llvm::SelectionDAG::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::isa(), isCMN(), isLegalArithImmed(), isLegalCmpImmed(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), LHS, numberOfInstrToLoadImm(), RHS, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, shouldBeAdjustedToZero(), llvm::ISD::SIGN_EXTEND_INREG, std::swap(), and llvm::ISD::ZEXTLOAD.
Referenced by performSetccAddFolding().
|
static |
Definition at line 4255 of file AArch64ISelLowering.cpp.
References assert(), DL, FlagsVT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::AArch64CC::HS, LHS, llvm_unreachable, llvm::AArch64CC::LO, llvm::ISD::MUL, Mul, llvm::ISD::MULHS, llvm::ISD::MULHU, llvm::AArch64CC::NE, Opc, RHS, llvm::ISD::SADDO, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMULO, llvm::ISD::SRA, llvm::ISD::SSUBO, llvm::ISD::TRUNCATE, llvm::ISD::UADDO, llvm::ISD::UMULO, llvm::ISD::USUBO, llvm::AArch64CC::VS, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerXALUO().
|
static |
Definition at line 28502 of file AArch64ISelLowering.cpp.
References llvm::Acquire, llvm::AcquireRelease, assert(), llvm_unreachable, llvm::Monotonic, llvm::Release, and llvm::SequentiallyConsistent.
Referenced by ReplaceATOMIC_LOAD_128Results().
Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
Definition at line 4055 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::dyn_cast(), Opc, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, and llvm::ISD::SRL.
Referenced by getAArch64Cmp().
|
static |
Like SelectionDAG::getCondCode(), but for AArch64 condition codes.
Definition at line 3628 of file AArch64ISelLowering.cpp.
References CondCodeVT, and llvm::SelectionDAG::getConstant().
Referenced by carryFlagToValue(), emitConditionalComparison(), foldADCToCINC(), foldCSELOfCSEL(), getAArch64Cmp(), getPTest(), getSETCC(), LowerBRCOND(), LowerXALUO(), overflowFlagToValue(), performAddCSelIntoCSinc(), performANDORCSELCombine(), performANDSETCCCombine(), performBRCONDCombine(), performCSELCombine(), performRNDRCombine(), performSETCCCombine(), and reassociateCSELOperandsForCSE().
Definition at line 5485 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), and llvm::EVT::isFixedLengthVector().
|
static |
Definition at line 29655 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by combineToVCPOP(), convertFixedMaskToScalableVector(), GenerateFixedLengthSVETBL(), getContainerForFixedLengthVector(), getVCIXISDNodeWCHAIN(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaVID(), lowerCttzElts(), lowerFixedVectorSegLoadIntrinsics(), lowerFixedVectorSegStoreIntrinsics(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), lowerScalarInsert(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), LowerVectorMatch(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), lowerVZIP(), matchSplatAsGather(), performFP_TO_INTCombine(), and processVCIXOperands().
|
static |
Definition at line 21432 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, llvm::isNullConstant(), llvm::isOneConstant(), and llvm::AArch64CC::NV.
Referenced by foldOverflowCheck(), and performBRCONDCombine().
Definition at line 14318 of file AArch64ISelLowering.cpp.
References llvm_unreachable.
Referenced by performDUPCombine().
|
static |
Definition at line 12671 of file AArch64ISelLowering.cpp.
References llvm::EVT::getFltSemantics(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::Log2_64_Ceil(), SDValue(), llvm::APFloatBase::semanticsPrecision(), and llvm::TargetLoweringBase::Unspecified.
getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction.
Definition at line 13342 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSizeInBits(), and llvm::EVT::getVectorElementType().
Referenced by GeneratePerfectShuffle(), and llvm::AArch64TargetLowering::ReconstructShuffle().
Definition at line 6776 of file AArch64ISelLowering.cpp.
References llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key.
Referenced by performGLD1Combine().
Definition at line 7966 of file AArch64ISelLowering.cpp.
References llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::Intrinsic::not_intrinsic.
Referenced by llvm::AMDGPURegisterBankInfo::applyMappingImpl(), collectBitParts(), llvm::SITargetLowering::CollectTargetIntrinsicOperands(), FindPreallocatedCall(), llvm::AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic(), llvm::AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects(), llvm::AArch64RegisterBankInfo::getInstrMapping(), llvm::AMDGPURegisterBankInfo::getInstrMapping(), llvm::VPRecipeBuilder::handleReplication(), llvm::SITargetLowering::isCanonicalized(), isFPIntrinsic(), isFPIntrinsic(), llvm::AArch64TargetLowering::isReassocProfitable(), IsSVECntIntrinsic(), llvm::MipsLegalizerInfo::legalizeIntrinsic(), performIntrinsicCombine(), llvm::SubsumingPositionIterator::SubsumingPositionIterator(), tryConvertSVEWideCompare(), llvm::SPIRVGlobalRegistry::updateAssignType(), and valueIsKnownNeverF32Denorm().
|
inlinestatic |
Definition at line 207 of file AArch64ISelLowering.cpp.
References llvm_unreachable.
Definition at line 182 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSimpleVT(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by combineSVEReductionInt(), LowerVectorMatch(), performDupLane128Combine(), and tryToReplaceScalarFPConversionWithSVE().
|
static |
Definition at line 29682 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::AArch64Subtarget::getMaxSVEVectorSizeInBits(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), getPTrue(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::getSVEPredPatternFromNumElements(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by convertFixedMaskToScalableVector(), and getPredicateForVector().
|
static |
Definition at line 29727 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::changeVectorElementType(), DL, getPTrue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isScalableVector(), and llvm::TargetLoweringBase::isTypeLegal().
Referenced by getPredicateForVector().
|
static |
Definition at line 29735 of file AArch64ISelLowering.cpp.
References DL, getPredicateForFixedLengthVector(), getPredicateForScalableVector(), and llvm::EVT::isFixedLengthVector().
Referenced by tryCombineExtendRShTrunc().
|
static |
Definition at line 12869 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorElementType(), llvm::EVT::isScalableVector(), llvm_unreachable, Upa, Uph, and Upl.
Definition at line 222 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::isScalableVector(), and llvm_unreachable.
|
static |
Definition at line 22526 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::ANY_ACTIVE, assert(), Cond, DL, llvm::AArch64CC::FIRST_ACTIVE, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), getSVEPredicateBitCast(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTypeToTransformTo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::TargetLoweringBase::isTypeLegal(), isZeroingInactiveLanes(), llvm::AArch64CC::NONE_ACTIVE, and llvm::Test.
Referenced by performFirstTrueTestVectorCombine(), performIntrinsicCombine(), and performLastTrueTestVectorCombine().
|
inlinestatic |
Definition at line 5829 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and llvm::SelectionDAG::getTargetConstant().
Referenced by getPredicateForFixedLengthVector(), getPredicateForScalableVector(), LowerFLDEXP(), optimizeIncrementingWhile(), performFirstTrueTestVectorCombine(), performLastTrueTestVectorCombine(), and performUnpackCombine().
|
static |
Definition at line 12900 of file AArch64ISelLowering.cpp.
References llvm::EVT::getFixedSizeInBits(), llvm::EVT::isScalarInteger(), llvm_unreachable, Uci, and Ucj.
|
static |
Definition at line 16584 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), and llvm::SDValue::getValueType().
|
static |
Definition at line 26716 of file AArch64ISelLowering.cpp.
References assert(), llvm::BitWidth, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::Log2_32(), llvm::Offset, llvm::ISD::SHL, and llvm::ISD::SPLAT_VECTOR.
Referenced by performGatherLoadCombine(), and performScatterStoreCombine().
|
static |
Helper function to create 'CSET', which is equivalent to 'CSINC <Wd>, WZR, WZR, invert(<cond>)'.
Definition at line 12940 of file AArch64ISelLowering.cpp.
References DL, getCondCode(), llvm::SelectionDAG::getConstant(), and llvm::SelectionDAG::getNode().
Referenced by combineAnd(), combineCMov(), combineM68kSetCC(), combineOr(), combinePredicateReduction(), combineSetCC(), combineSubSetcc(), combineVectorSizedSetCCEquality(), combineX86SetCC(), foldXor1SetCC(), LowerADDSUBO_CARRY(), llvm::X86TargetLowering::LowerAsmOutputForConstraint(), LowerCMP_SWAP(), LowerINTRINSIC_W_CHAIN(), LowerPARITY(), LowerXALUO(), and llvm::X86TargetLowering::ReplaceNodeResults().
Definition at line 6799 of file AArch64ISelLowering.cpp.
References llvm_unreachable.
Referenced by performGLD1Combine().
|
static |
Definition at line 8999 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::CallLoweringInfo::Callee, llvm::TargetLowering::CallLoweringInfo::CB, llvm::dyn_cast(), and llvm::SMEAttrs::Normal.
|
static |
Definition at line 9328 of file AArch64ISelLowering.cpp.
References llvm::AArch64SME::Always, llvm::SMECallAttrs::callee(), llvm::SMECallAttrs::caller(), llvm::SMEAttrs::hasNonStreamingInterface(), llvm::SMEAttrs::hasStreamingBody(), llvm::SMEAttrs::hasStreamingCompatibleInterface(), llvm::SMEAttrs::hasStreamingInterface(), llvm::AArch64SME::IfCallerIsNonStreaming, llvm::AArch64SME::IfCallerIsStreaming, and llvm_unreachable.
|
static |
Definition at line 17751 of file AArch64ISelLowering.cpp.
References assert(), and llvm::Intrinsic::getOrInsertDeclaration().
Referenced by llvm::AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(), and llvm::AArch64TargetLowering::lowerInterleavedLoad().
|
static |
Definition at line 17768 of file AArch64ISelLowering.cpp.
References assert(), and llvm::Intrinsic::getOrInsertDeclaration().
Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore(), and llvm::AArch64TargetLowering::lowerInterleaveIntrinsicToStore().
|
static |
Definition at line 17723 of file AArch64ISelLowering.cpp.
References llvm::ScalableVectorType::get(), llvm::Type::getBFloatTy(), llvm::Type::getContext(), llvm::Type::getDoubleTy(), llvm::VectorType::getElementType(), llvm::Type::getFloatTy(), llvm::Type::getHalfTy(), llvm::Type::getInt16Ty(), llvm::Type::getInt32Ty(), llvm::Type::getInt64Ty(), llvm::Type::getInt8Ty(), and llvm_unreachable.
Referenced by llvm::AArch64TargetLowering::lowerInterleavedLoad(), and llvm::AArch64TargetLowering::lowerInterleavedStore().
Definition at line 23287 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getSimpleVT(), llvm::EVT::isSimple(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by performGatherLoadCombine(), performLD1Combine(), performScatterStoreCombine(), and performST1Combine().
|
static |
Definition at line 5889 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::EVT::bitsGT(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), and isZeroingInactiveLanes().
Referenced by getPTest().
|
static |
Definition at line 26281 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::CallingConv::C, llvm::dyn_cast(), getTestBitOperand(), getValueType(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::XOR.
Referenced by getTestBitOperand(), and performTBZCombine().
|
static |
Definition at line 16592 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), getVectorBitwiseReduce(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::EVT::isFixedLengthVector(), llvm::EVT::isPow2VectorType(), llvm_unreachable, llvm::Lo, llvm::ISD::OR, SDValue(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::SplitVector(), llvm::ISD::SRL, and llvm::ISD::XOR.
Referenced by getVectorBitwiseReduce(), and performVecReduceBitwiseCombine().
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value.
Definition at line 16337 of file AArch64ISelLowering.cpp.
References llvm::dyn_cast(), llvm::APInt::getSExtValue(), and llvm::BuildVectorSDNode::isConstantSplat().
Referenced by isVShiftLImm(), isVShiftLImm(), isVShiftRImm(), and isVShiftRImm().
|
static |
Definition at line 8078 of file AArch64ISelLowering.cpp.
References llvm::MachineFrameInfo::CreateSpillStackObject(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getFrameIndex(), llvm::TargetLoweringBase::getFrameIndexTy(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::AArch64FunctionInfo::getZT0SpillSlotIndex(), llvm::AArch64FunctionInfo::hasZT0SpillSlotIndex(), and llvm::AArch64FunctionInfo::setZT0SpillSlotIndex().
bool hasNearbyPairedStore | ( | Iter | It, |
Iter | End, | ||
Value * | Ptr, | ||
const DataLayout & | DL ) |
Definition at line 17931 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), Ptr, and llvm::APInt::sextOrTrunc().
Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore().
Definition at line 20349 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::FADD, and llvm::ISD::STRICT_FADD.
Referenced by performExtractVectorEltCombine().
if | ( | MST-> | isTruncatingStore() | ) |
Definition at line 24947 of file AArch64ISelLowering.cpp.
References DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::MaskedStoreSDNode::getOffset(), SDValue(), and trySimplifySrlAddToRshrnb().
|
static |
Definition at line 5546 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isSignExtended(), N, and llvm::ISD::SUB.
Referenced by LowerMUL(), and selectUmullSmull().
|
static |
Definition at line 5557 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isZeroExtended(), N, and llvm::ISD::SUB.
Referenced by LowerMUL(), and selectUmullSmull().
|
static |
Definition at line 15022 of file AArch64ISelLowering.cpp.
References llvm::getNumElementsFromSVEPredPattern(), llvm::SelectionDAG::getSubtarget(), llvm::ISD::isConstantSplatVectorAllOnes(), N, and llvm::AArch64::SVEBitsPerBlock.
Referenced by convertMergedOpToPredOp(), foldCSELofLASTB(), instCombineSVECmpNE(), instCombineSVELD1(), instCombineSVESel(), instCombineSVESrshl(), instCombineSVEST1(), instCombineSVEUxt(), instCombineSVEVectorBinOp(), performSetccMergeZeroCombine(), performSVEAndCombine(), performVSelectCombine(), simplifySVEIntrinsic(), and tryLowerToSLI().
Definition at line 14997 of file AArch64ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), and llvm::ConstantSDNode::getZExtValue().
Referenced by isPow2Splat().
Definition at line 15014 of file AArch64ISelLowering.cpp.
References llvm::ISD::isConstantSplatVectorAllZeros(), and N.
Referenced by performVSelectCombine().
Definition at line 23027 of file AArch64ISelLowering.cpp.
References llvm::ISD::isConstantSplatVectorAllZeros(), and N.
Referenced by performSignExtendSetCCCombine().
|
static |
Definition at line 3680 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::isKnownNeverZero(), llvm::isNullConstant(), isSafeSignedCMN(), and llvm::ISD::SUB.
Referenced by emitComparison(), emitConditionalComparison(), and getAArch64Cmp().
Definition at line 21425 of file AArch64ISelLowering.cpp.
Referenced by foldCSELOfCSEL(), foldOverflowCheck(), and performBRCONDCombine().
Definition at line 14030 of file AArch64ISelLowering.cpp.
References E(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), I, and llvm::Offset.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal(), and tryFormConcatFromShuffle().
Definition at line 20060 of file AArch64ISelLowering.cpp.
References llvm::dyn_cast(), llvm::EVT::getSimpleVT(), llvm::EVT::getVectorElementType(), llvm::EVT::isSimple(), N, llvm::MVT::SimpleTy, and llvm::ISD::SPLAT_VECTOR.
|
static |
Definition at line 25366 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::NV, llvm::AArch64CC::PL, llvm::ISD::SEXTLOAD, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.
Referenced by performCONDCombine().
Definition at line 21054 of file AArch64ISelLowering.cpp.
References llvm::ISD::EXTRACT_SUBVECTOR, and N.
Referenced by performAddSubLongCombine(), tryCombineLongOpWithDup(), and tryCombineMULLWithUZP1().
|
static |
Definition at line 5498 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, llvm::dyn_cast(), llvm::EVT::getScalarSizeInBits(), llvm::isIntN(), isSigned(), llvm::isUIntN(), and N.
Referenced by isSignExtended(), isSignExtended(), isZeroExtended(), and isZeroExtended().
Definition at line 21765 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::cast(), llvm::dyn_cast(), llvm::ConstantSDNode::getZExtValue(), llvm::isa(), llvm::ISD::isExtOpcode(), N, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, and llvm::ISD::SRL.
Referenced by performAddCombineSubShift().
Definition at line 13897 of file AArch64ISelLowering.cpp.
References llvm::find_if(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), and llvm::APInt::logBase2().
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
Definition at line 23696 of file AArch64ISelLowering.cpp.
References llvm::SDNode::getValueType(), and N.
Referenced by performUzpCombine().
Definition at line 13991 of file AArch64ISelLowering.cpp.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
Definition at line 2405 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, llvm::dyn_cast(), and N.
Definition at line 27177 of file AArch64ISelLowering.cpp.
Referenced by removeRedundantInsertVectorElt().
Definition at line 3633 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, llvm::dbgs(), and LLVM_DEBUG.
Referenced by getAArch64Cmp(), llvm::AArch64TargetLowering::isLegalAddImmediate(), and isLegalCmpImmed().
Definition at line 3641 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, and isLegalArithImmed().
Referenced by getAArch64Cmp().
|
static |
Definition at line 21978 of file AArch64ISelLowering.cpp.
References B(), llvm::ISD::BUILD_VECTOR, llvm::cast(), llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::hasOneUse(), llvm::SDValue::hasOneUse(), I, llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::VECTOR_SHUFFLE.
Referenced by areLoadedOffsetButOtherwiseSame(), and performExtBinopLoadFold().
Definition at line 253 of file AArch64ISelLowering.cpp.
References Opc.
Referenced by performVSelectCombine().
Definition at line 21335 of file AArch64ISelLowering.cpp.
References llvm::isNullConstant(), and llvm::ISD::SUB.
Referenced by performNegCMovCombine(), and performNegCSelCombine().
Definition at line 23777 of file AArch64ISelLowering.cpp.
References SDValue().
Referenced by performUzpCombine().
Definition at line 2425 of file AArch64ISelLowering.cpp.
References isIntImmediate(), N, and Opc.
|
static |
Definition at line 11433 of file AArch64ISelLowering.cpp.
References isOrXorChain(), MaxXors, N, llvm::ISD::OR, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by isOrXorChain(), and performOrXorChainCombine().
|
inlinestatic |
Returns true if VT's elements occupy the lowest bit positions of its associated register class without any intervening space.
For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the same register class, but only nxv8f16 can be treated as a packed vector.
Definition at line 244 of file AArch64ISelLowering.cpp.
References assert(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), and llvm::AArch64::SVEBitsPerBlock.
Definition at line 8073 of file AArch64ISelLowering.cpp.
References llvm::EVT::isFixedLengthVector(), llvm::EVT::isFloatingPoint(), and llvm::EVT::isScalableVector().
Definition at line 16205 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::isa(), isAllConstantBuildVector(), llvm::isPowerOf2_64(), and llvm::ISD::SPLAT_VECTOR.
Definition at line 20364 of file AArch64ISelLowering.cpp.
References llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::ISD::SETCC.
Referenced by performFirstTrueTestVectorCombine().
|
static |
Definition at line 3654 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::getSignedMinValue(), and llvm::APInt::isMinSignedValue().
Referenced by isCMN().
|
static |
Check whether or not Op
is a SET_CC operation, either a generic or an AArch64 lowered one.
SetCCInfo
is filled accordingly.
Definition at line 21098 of file AArch64ISelLowering.cpp.
References SetCCInfo::AArch64, llvm::cast(), AArch64SetCCInfo::CC, GenericSetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::dyn_cast(), SetCCInfo::Generic, llvm::AArch64CC::getInvertedCondCode(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, llvm::ISD::SETCC, and std::swap().
Referenced by isSetCCOrZExtSetCC().
|
static |
Definition at line 21141 of file AArch64ISelLowering.cpp.
References Info, isSetCC(), and llvm::ISD::ZERO_EXTEND.
Referenced by performSetccAddFolding().
|
static |
Definition at line 5534 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::SIGN_EXTEND.
Referenced by llvm::PPCTargetLowering::EmitPartwordAtomicBinary(), isAddSubSExt(), isAddSubSExt(), LowerMUL(), performMulCombine(), and selectUmullSmull().
Definition at line 26202 of file AArch64ISelLowering.cpp.
References llvm::BitWidth.
Referenced by performSetccMergeZeroCombine().
Definition at line 13741 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements().
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
Definition at line 19266 of file AArch64ISelLowering.cpp.
References getIntrinsicID(), and llvm::SDValue::getNode().
Referenced by performMulCombine().
isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
Definition at line 13978 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements().
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
Definition at line 13959 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements().
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
Check if the value of Offset
represents a valid immediate for the SVE gather load/prefetch and scatter store instructiona with vector base and immediate offset addressing mode:
[<Zn>.[S|D]{, #<imm>}]
where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
Definition at line 26754 of file AArch64ISelLowering.cpp.
References llvm::dyn_cast(), llvm::ConstantSDNode::getZExtValue(), isValidImmForSVEVecImmAddrMode(), and llvm::Offset.
|
inlinestatic |
Check if the value of OffsetInBytes
can be used as an immediate for the gather load/prefetch and scatter store instructions with vector base and immediate offset addressing mode:
[<Zn>.[S|D]{, #<imm>}]
where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
Definition at line 26734 of file AArch64ISelLowering.cpp.
Referenced by combineSVEPrefetchVecBaseImmOff(), isValidImmForSVEVecImmAddrMode(), performGatherLoadCombine(), and performScatterStoreCombine().
Definition at line 2413 of file AArch64ISelLowering.cpp.
Referenced by performConcatVectorsCombine().
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation.
That value must be in the range: 0 <= Value < ElementBits for a left shift; or 0 <= Value <= ElementBits for a long left shift.
Definition at line 16357 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().
Referenced by LowerShift(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), and PerformShiftCombine().
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation.
The value must be in the range: 1 <= Value <= ElementBits for a right shift; or
Definition at line 16368 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().
Referenced by LowerShift(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), and PerformShiftCombine().
Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vector lane type.
If that is the case the function returns true and writes the value of the DUP instruction lane operand into DupLaneOp
Definition at line 13824 of file AArch64ISelLowering.cpp.
References assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), BlockSize, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), and I.
|
static |
Definition at line 5540 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::ZERO_EXTEND.
Referenced by isAddSubZExt(), isAddSubZExt(), LowerMUL(), performMulCombine(), and selectUmullSmull().
Definition at line 292 of file AArch64ISelLowering.cpp.
References llvm::ISD::INTRINSIC_WO_CHAIN, and llvm::ISD::SPLAT_VECTOR.
Referenced by getPTest(), and getSVEPredicateBitCast().
isZerosVector - Check whether SDNode N is a zero-filled vector.
Definition at line 3457 of file AArch64ISelLowering.cpp.
References llvm::ISD::isConstantSplatVectorAllZeros(), llvm::isNullConstant(), llvm::isNullFPConstant(), and N.
Referenced by GenerateTBL(), performAddDotCombine(), performInsertSubvectorCombine(), performSetccMergeZeroCombine(), performSetCCPunpkCombine(), performUnpackCombine(), and performVectorDeinterleaveCombine().
isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
Definition at line 13940 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements().
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
|
static |
Legalize the gather prefetch (scalar + vector addressing mode) when the offset vector is an unpacked 32-bit scalable vector. The other cases (Offset != nxv2i32) do not need legalization.
Definition at line 27133 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), N, llvm::Offset, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
Definition at line 10921 of file AArch64ISelLowering.cpp.
References llvm::cast(), llvm::EVT::getFixedSizeInBits(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::SIGN_EXTEND_INREG.
|
static |
Definition at line 7059 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getNode(), llvm::TargetMachine::getPointerSize(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTargetMachine(), llvm::SelectionDAG::getZeroExtendInReg(), N, llvm::ARM64AS::PTR32_SPTR, llvm::ARM64AS::PTR32_UPTR, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::LowerOperation(), llvm::X86TargetLowering::LowerOperation(), and llvm::X86TargetLowering::ReplaceNodeResults().
|
static |
Definition at line 4466 of file AArch64ISelLowering.cpp.
References carryFlagToValue(), DL, FlagsVT, llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), overflowFlagToValue(), SDValue(), and valueToCarryFlag().
Referenced by llvm::AArch64TargetLowering::LowerOperation().
|
static |
Definition at line 7376 of file AArch64ISelLowering.cpp.
References Cond, DL, emitConjunction(), getCondCode(), llvm::SelectionDAG::getNode(), and SDValue().
Referenced by llvm::AArch64TargetLowering::LowerOperation().
|
static |
Definition at line 7430 of file AArch64ISelLowering.cpp.
References llvm::EVT::changeVectorElementType(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getPTrue(), llvm::SelectionDAG::getUNDEF(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::INTRINSIC_WO_CHAIN, SDValue(), llvm::ISD::SIGN_EXTEND, and X.
Referenced by llvm::AArch64TargetLowering::LowerOperation().
|
static |
Definition at line 7394 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getConstant(), llvm::MVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), and SDValue().
Referenced by llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 4521 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), and llvm::SelectionDAG::getTargetConstant().
Referenced by llvm::AArch64TargetLowering::LowerOperation(), llvm::ARMTargetLowering::LowerOperation(), llvm::RISCVTargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 10821 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), DL, llvm::SelectionDAG::getMachineNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::isNullConstant(), llvm::report_fatal_error(), and SDValue().
SDValue LowerSMELdrStr | ( | SDValue | N, |
SelectionDAG & | DAG, | ||
bool | IsLoad ) |
Definition at line 5988 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::sampleprof::Base, llvm::CallingConv::C, llvm::cast(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::isa(), llvm::ISD::MUL, Mul, N, SDValue(), and llvm::ISD::SIGN_EXTEND.
|
static |
Definition at line 22422 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 22436 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::ElementCount::getScalable(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::EVT::isScalableVector(), llvm::ISD::MUL, N, SDValue(), and llvm::AArch64::SVEBitsPerBlock.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 22406 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::sampleprof::Base, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStepVector(), llvm::SDValue::getValueType(), llvm::ISD::MUL, Mul, N, and llvm::ISD::SPLAT_VECTOR.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 7028 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isVector(), and llvm::ISD::TRUNCATE.
SDValue LowerVectorMatch | ( | SDValue | Op, |
SelectionDAG & | DAG ) |
Definition at line 6035 of file AArch64ISelLowering.cpp.
References assert(), convertFixedMaskToScalableVector(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::EVT::getFixedSizeInBits(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getSplatVector(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::is128BitVector(), llvm::EVT::isScalableVector(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::TRUNCATE.
|
static |
Definition at line 4491 of file AArch64ISelLowering.cpp.
References DL, getAArch64XALUOOp(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::isTypeLegal(), and SDValue().
Referenced by LowerMULO(), llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().
|
static |
Return true if we might ever do TCO for calls with this calling convention.
Definition at line 8908 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::AArch64_SVE_VectorCall, llvm::CallingConv::C, llvm::CallingConv::Fast, llvm::CallingConv::PreserveAll, llvm::CallingConv::PreserveMost, llvm::CallingConv::PreserveNone, llvm::CallingConv::Swift, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.
Referenced by llvm::AArch64CallLowering::isEligibleForTailCallOptimization().
|
static |
Definition at line 15306 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::ISD::BUILD_VECTOR, DL, llvm::dyn_cast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), and llvm::EVT::isFloatingPoint().
Definition at line 3647 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, llvm::AArch64_IMM::expandMOVImm(), and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by getAArch64Cmp().
|
static |
Definition at line 5837 of file AArch64ISelLowering.cpp.
References DL, llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), getPTrue(), llvm::SelectionDAG::getSubtarget(), llvm::getSVEPredPatternFromNumElements(), llvm::APInt::getZExtValue(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isa(), N, llvm::APInt::sadd_ov(), SDValue(), llvm::APInt::uadd_ov(), X, and Y.
Referenced by performActiveLaneMaskCombine().
|
static |
Definition at line 2431 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::TargetLoweringOpt::CombineTo(), llvm::TargetLowering::TargetLoweringOpt::DAG, DL, llvm::AArch64_AM::encodeLogicalImmediate(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::APInt::getZExtValue(), llvm::Hi, llvm::AArch64_AM::isLogicalImmediate(), llvm::isShiftedMask_64(), SDValue(), and Size.
Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant().
|
static |
Definition at line 4455 of file AArch64ISelLowering.cpp.
References assert(), DL, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getResNo(), and llvm::AArch64CC::VS.
Referenced by lowerADDSUBO_CARRY().
|
static |
Definition at line 12916 of file AArch64ISelLowering.cpp.
References llvm::StringSwitch< T, R >::Case(), Cond, llvm::StringSwitch< T, R >::Default(), llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.
Referenced by llvm::X86TargetLowering::getConstraintType(), llvm::X86TargetLowering::getRegForInlineAsmConstraint(), and llvm::X86TargetLowering::LowerAsmOutputForConstraint().
|
static |
Definition at line 12860 of file AArch64ISelLowering.cpp.
References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), Upa, Uph, and Upl.
|
static |
Definition at line 12892 of file AArch64ISelLowering.cpp.
References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), Uci, and Ucj.
|
static |
Definition at line 12837 of file AArch64ISelLowering.cpp.
References llvm::StringRef::drop_front(), llvm::StringRef::ends_with(), llvm::StringRef::getAsInteger(), llvm::StringRef::size(), llvm::StringRef::starts_with(), and llvm::StringRef::substr().
|
static |
Definition at line 18857 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::divideCoefficientBy(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getElementCount(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), I, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, llvm::Offset, optimizeIncrementingWhile(), SDValue(), llvm::ISD::UADDSAT, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21818 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), isOpcWithIntImmediate(), LHS, N, performAddCombineSubShift(), RHS, SDValue(), and llvm::ISD::SHL.
Referenced by performAddSubCombine().
|
static |
Definition at line 21790 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), llvm::isa(), isExtendOrShiftOperand(), N, SDValue(), llvm::ISD::SUB, and Y.
Referenced by performAddCombineForShiftedOperands().
|
static |
Perform the scalar expression combine in the form of: CSEL(c, 1, cc) + b => CSINC(b+c, b, cc) CSNEG(c, -1, cc) + b => CSINC(b+c, b, cc)
Definition at line 21240 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::CallingConv::C, llvm::cast(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getAllOnesConstant(), llvm::ConstantSDNode::getAPIntValue(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ConstantSDNode::isAllOnes(), llvm::TargetLoweringBase::isLegalAddImmediate(), llvm::ConstantSDNode::isOne(), llvm::EVT::isScalarInteger(), LHS, N, RHS, SDValue(), and std::swap().
Referenced by performAddSubCombine().
|
static |
Definition at line 21313 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), isZerosVector(), N, SDValue(), and std::swap().
Referenced by performAddSubCombine().
|
static |
Definition at line 22222 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, N, performAddCombineForShiftedOperands(), performAddCSelIntoCSinc(), performAddDotCombine(), performAddSubIntoVectorOp(), performAddSubLongCombine(), performAddUADDVCombine(), performExtBinopLoadFold(), performNegCSelCombine(), performSubAddMULCombine(), performSVEMulAddSubCombine(), and performVectorExtCombine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21944 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::TargetLoweringBase::isOperationExpand(), N, llvm::ISD::SCALAR_TO_VECTOR, and SDValue().
Referenced by performAddSubCombine().
|
static |
Definition at line 21383 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getNode(), llvm::MVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, performSetccAddFolding(), RHS, SDValue(), llvm::ISD::SIGN_EXTEND, tryExtendDUPToExtractHigh(), and llvm::ISD::ZERO_EXTEND.
Referenced by performAddSubCombine().
|
static |
Definition at line 21201 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::isScalarInteger(), LHS, N, RHS, and SDValue().
Referenced by performAddSubCombine().
|
static |
Definition at line 20250 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dyn_cast(), llvm::APInt::getBitWidth(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), I, llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, performANDORCSELCombine(), performANDSETCCCombine(), performSVEAndCombine(), resolveBuildVector(), RHS, SDValue(), tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), llvm::KnownBits::Zero, and llvm::APInt::zext().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().
|
static |
Definition at line 19982 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::dyn_cast(), FlagsVT, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::isNullConstant(), llvm::isOneConstant(), N, SDValue(), and std::swap().
Referenced by performANDCombine(), and performORCombine().
|
static |
Definition at line 20211 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, DL, emitConjunction(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, SDValue(), llvm::ISD::SELECT, and llvm::ISD::SETCC.
Referenced by performANDCombine().
|
static |
Definition at line 25569 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::AArch64CC::EQ, llvm::SDNode::getAsZExtVal(), getCondCode(), getCSETCondCode(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::Function::hasFnAttribute(), llvm::isa(), isCMP(), llvm::isNullConstant(), llvm::isOneConstant(), LHS, N, llvm::AArch64CC::NE, performCONDCombine(), RHS, SDValue(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 27282 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::isScalableVector(), N, llvm::ISD::OR, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Combines a buildvector(sext/zext) or shuffle(sext/zext, undef) node pattern into sext/zext(buildvector) or sext/zext(shuffle) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
Definition at line 19332 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::ISD::BUILD_VECTOR, calculatePreExtendType(), llvm::cast(), llvm::EVT::changeVectorElementType(), DL, llvm::drop_begin(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::isUndef(), Opc, llvm::SDNode::ops(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::VECTOR_SHUFFLE, and llvm::ISD::ZERO_EXTEND.
Referenced by performMulVectorExtendCombine().
|
static |
Definition at line 21490 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::EVT::changeVectorElementType(), Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP_ROUND, llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorMinNumElements(), llvm::isa(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::TargetLoweringBase::isTypeLegal(), N, SDValue(), and llvm::ISD::UNDEF.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20618 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::Add, llvm::all_of(), assert(), llvm::cast(), llvm::ISD::CONCAT_VECTORS, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dbgs(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SDNode::hasOneUse(), llvm::EVT::is128BitVector(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isBinOp(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::SDNode::isOnlyUserOf(), llvm::EVT::isScalableVector(), llvm::SDValue::isUndef(), llvm::MVT::isVector(), isVectorizedBinOp(), LLVM_DEBUG, N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, SDValue(), llvm::ISD::TRUNCATE, WidenVector(), X, llvm::ISD::XOR, and Y.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 25490 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::ISD::AND, llvm::cast(), checkValueWidth(), llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDNode::hasOneUse(), llvm::isa(), isEquivalentMaskless(), N, performSubsToAndsCombine(), llvm::SelectionDAG::ReplaceAllUsesWith(), and SDValue().
Referenced by performBRCONDCombine(), and performCSELCombine().
|
static |
Definition at line 25920 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), Cond, DL, llvm::SelectionDAG::doesNodeExist(), llvm::AArch64CC::EQ, foldCSELOfCSEL(), foldCSELofCTTZ(), foldCSELofLASTB(), getCondCode(), llvm::SelectionDAG::getNode(), getSwappedCondition(), llvm::SDValue::getValue(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::isNullConstant(), llvm::AArch64CC::MI, N, llvm::AArch64CC::NE, performCONDCombine(), llvm::AArch64CC::PL, reassociateCSELOperandsForCSE(), SDValue(), llvm::ISD::SUB, and llvm::Sub.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 26703 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITREVERSE, llvm::ISD::CTTZ, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::isScalarInteger(), N, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 26575 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), getDUPLANEOp(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), N, performPostLD1Combine(), and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 27302 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), N, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 22104 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, areLoadedOffsetButOtherwiseSame(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::Hi, llvm::ISD::isConstantSplatVector(), llvm::ISD::isExtOpcode(), llvm::EVT::isFixedLengthVector(), isLoadOrMultipleLoads(), llvm::TargetLoweringBase::isTypeLegal(), llvm::Lo, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::SDNode::op_values(), llvm::Other, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::ISD::SHL, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::SUB, std::swap(), and llvm::zip().
Referenced by performAddSubCombine().
|
static |
Definition at line 23193 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ANY_EXTEND, llvm::ISD::BSWAP, DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, llvm::Intrinsic::not_intrinsic, performSignExtendSetCCCombine(), performZExtDeinterleaveShuffleCombine(), performZExtUZPCombine(), SDValue(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, tryCombineLongOpWithDup(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20457 of file AArch64ISelLowering.cpp.
References llvm::ArrayRef(), assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::is_contained(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLoweringBase::isOperationLegal(), N, and SDValue().
Referenced by performExtractVectorEltCombine().
|
static |
Definition at line 20880 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::EVT::getVectorElementType(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isScalableVector(), N, SDValue(), and llvm::ISD::SPLAT_VECTOR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20484 of file AArch64ISelLowering.cpp.
References llvm::all_of(), llvm::ISD::ANY_EXTEND, assert(), llvm::commonAlignment(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::dyn_cast(), llvm::ISD::EXTLOAD, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getExtLoad(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getObjectPtrOffset(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), hasPairwiseAdd(), llvm::ISD::INSERT_SUBVECTOR, llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isInteger(), llvm::AArch64Subtarget::isLittleEndian(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::SDNode::isStrictFPOpcode(), llvm::SDValue::isUndef(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::Offset, llvm::Other, performExtractLastActiveCombine(), performFirstTrueTestVectorCombine(), performLastTrueTestVectorCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::SDNode::uses(), llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20313 of file AArch64ISelLowering.cpp.
References A(), B(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::FADD, llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, LHS, N, Opc, RHS, SDValue(), and llvm::SDNode::setFlags().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20393 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::AArch64CC::FIRST_ACTIVE, getPTest(), getPTrue(), llvm::SDValue::getResNo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::isNullConstant(), isPredicateCCSettingOp(), llvm::EVT::isScalableVector(), N, and SDValue().
Referenced by performExtractVectorEltCombine().
|
static |
Definition at line 26142 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, Generic, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getVTList(), LHS, N, RHS, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 27244 of file AArch64ISelLowering.cpp.
References llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::EXTLOAD, llvm::ISD::FP_ROUND, llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getExtLoad(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MemSDNode::getMemOperand(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isNormalLoad(), N, SDValue(), and llvm::AArch64Subtarget::useSVEForFixedLengthVectors().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.
Definition at line 19899 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, llvm::cast(), DL, llvm::ISD::FMUL, llvm::ISD::FP_TO_SINT, llvm::ISD::FP_TO_SINT_SAT, llvm::ISD::FP_TO_UINT_SAT, llvm::SelectionDAG::getConstant(), llvm::BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isa(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::TargetLoweringBase::isTypeLegal(), N, SDValue(), llvm::ISD::TRUNCATE, and tryToReplaceScalarFPConversionWithSVE().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 26874 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), isValidImmForSVEVecImmAddrMode(), N, llvm::Offset, SDValue(), llvm::AArch64::SVEBitsPerBlock, std::swap(), and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23972 of file AArch64ISelLowering.cpp.
References llvm::sampleprof::Base, llvm::cast(), DL, getGatherVecOpcode(), llvm::SelectionDAG::getNode(), getSignExtendedGatherOpcode(), llvm::EVT::getVectorElementType(), llvm::VTSDNode::getVT(), N, llvm::Offset, Opc, Scaled, SDValue(), and Signed.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 26652 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::CallingConv::C, llvm::cast(), llvm::AArch64Subtarget::ClassifyGlobalReference(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::GlobalValue::getDataLayout(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getGlobalAddress(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::DataLayout::getTypeAllocSize(), llvm::GlobalValue::getValueType(), llvm::AArch64II::MO_NO_FLAG, N, llvm::Offset, SDValue(), llvm::ISD::SUB, T, and llvm::SDNode::users().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20903 of file AArch64ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::EVT::isFixedLengthVector(), llvm::EVT::isInteger(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), isZerosVector(), llvm::Lo, N, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 27237 of file AArch64ISelLowering.cpp.
References N, performPostLD1Combine(), and removeRedundantInsertVectorElt().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 22686 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ADD, llvm::ISD::AND, llvm::AArch64CC::ANY_ACTIVE, combineAcrossLanesIntrinsic(), combineSVEBitSel(), combineSVEReductionFP(), combineSVEReductionInt(), combineSVEReductionOrderedFP(), convertMergedOpToPredOp(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::AArch64CC::FIRST_ACTIVE, llvm::SelectionDAG::getCondCode(), getIntrinsicID(), llvm::SelectionDAG::getNode(), getPTest(), llvm::AArch64CC::LAST_ACTIVE, LowerSVEIntrinsicDUP(), LowerSVEIntrinsicEXT(), LowerSVEIntrinsicIndex(), N, llvm::ISD::OR, llvm::ISD::SADDSAT, SDValue(), llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUO, llvm::ISD::SPLAT_VECTOR, llvm::ISD::SSUBSAT, llvm::ISD::SUB, tryCombineCRC32(), tryCombineFixedPointConvert(), tryCombineLongOpWithDup(), tryCombineNeonFcvtFP16ToI16(), tryCombineShiftImm(), tryConvertSVEWideCompare(), llvm::ISD::UADDSAT, llvm::ISD::USUBSAT, llvm::ISD::VSELECT, and llvm::ISD::XOR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19852 of file AArch64ISelLowering.cpp.
References llvm::cast(), llvm::MemSDNode::getAlign(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SDValue::hasOneUse(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::ISD::isNormalLoad(), N, performVectorCompareAndMaskUnaryOpCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), llvm::ISD::SINT_TO_FP, and tryToReplaceScalarFPConversionWithSVE().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20423 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPTest(), getPTrue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isScalableVector(), llvm::AArch64CC::LAST_ACTIVE, N, and SDValue().
Referenced by performExtractVectorEltCombine().
|
static |
Definition at line 23315 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isInteger(), N, Opc, SDValue(), llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23367 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::isFloatingPoint(), N, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23341 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isFloatingPoint(), N, llvm::ISD::NON_EXTLOAD, and llvm::ISD::UNINDEXED.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 25084 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::Data, DL, llvm::dyn_cast(), findMoreOptimalIndexType(), llvm::MaskedGatherScatterSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedGatherSDNode::getExtensionType(), llvm::MaskedGatherScatterSDNode::getIndex(), llvm::MaskedGatherScatterSDNode::getIndexType(), llvm::MaskedGatherScatterSDNode::getMask(), llvm::SelectionDAG::getMaskedGather(), llvm::SelectionDAG::getMaskedHistogram(), llvm::SelectionDAG::getMaskedScatter(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::MaskedGatherSDNode::getPassThru(), llvm::MaskedGatherScatterSDNode::getScale(), llvm::MaskedScatterSDNode::getValue(), llvm::SelectionDAG::getVTList(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::MaskedScatterSDNode::isTruncatingStore(), N, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19507 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::Add, llvm::APInt::ashr(), llvm::BitWidth, llvm::CallingConv::C, llvm::cast(), llvm::APInt::countr_zero(), DL, llvm::dyn_cast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::APInt::getZExtValue(), llvm::SDNode::hasOneUse(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::APInt::isNegative(), llvm::APInt::isNonNegative(), llvm::APInt::isPowerOf2(), isSignExtended(), IsSVECntIntrinsic(), isZeroExtended(), llvm::APInt::logBase2(), llvm::ISD::MUL, N, performMulVectorCmpZeroCombine(), performMulVectorExtendCombine(), performVectorExtCombine(), RHS, llvm::APInt::sdivrem(), SDValue(), llvm::APInt::sge(), llvm::ISD::SHL, llvm::APInt::sle(), llvm::ISD::SUB, llvm::Sub, std::swap(), llvm::ISD::TRUNCATE, and X.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 27472 of file AArch64ISelLowering.cpp.
References N, llvm::Intrinsic::not_intrinsic, SDValue(), tryCombineLongOpWithDup(), and tryCombineMULLWithUZP1().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19432 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::ISD::isConstantSplatVector(), llvm::APInt::isMask(), N, SDValue(), llvm::ISD::SETGT, and llvm::ISD::SRL.
Referenced by performMulCombine().
|
static |
Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
Definition at line 19412 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), Mul, performBuildShuffleExtendCombine(), and SDValue().
Referenced by performMulCombine().
|
static |
Definition at line 21346 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), isNegatedInteger(), N, and SDValue().
Referenced by performAddSubCombine().
|
static |
Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.
Definition at line 25126 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::ArrayRef(), llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::dyn_cast(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getResNo(), llvm::EVT::getSizeInBits(), llvm::Use::getUser(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm_unreachable, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), and llvm::SDNode::uses().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Get rid of unnecessary NVCASTs (that don't change the type).
Definition at line 26639 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), N, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20049 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, N, performANDORCSELCombine(), and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().
|
static |
Definition at line 11460 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::cast(), Cond, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), I, llvm::isNullConstant(), isOrXorChain(), LHS, N, llvm::ISD::OR, RHS, SDValue(), llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by performSETCCCombine().
|
static |
Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.
Definition at line 24076 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dyn_cast(), llvm::ISD::FMA, llvm::ISD::FMUL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getResNo(), llvm::EVT::getScalarSizeInBits(), llvm::Use::getUser(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullOrNullSplat(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::SDNode::uses(), and llvm::Vector.
Referenced by performDUPCombine(), and performInsertVectorEltCombine().
Definition at line 20087 of file AArch64ISelLowering.cpp.
References llvm::SDValue::getValueType(), N, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 27655 of file AArch64ISelLowering.cpp.
References A(), B(), DL, FlagsVT, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), N, and llvm::AArch64CC::NE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 27486 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), N, SDValue(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 26761 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isFloatingPoint(), llvm::MVT::isFloatingPoint(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), isValidImmForSVEVecImmAddrMode(), N, llvm::Offset, SDValue(), llvm::AArch64::SVEBitsPerBlock, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar. This replaces any scalar setcc in the above pattern with a vector one followed by a DUP shuffle on the result.
Definition at line 26510 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::changeVectorElementTypeToInteger(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFloatingPoint(), llvm::EVT::isScalableVT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LHS, N, RHS, llvm::ISD::SCALAR_TO_VECTOR, SDValue(), and llvm::ISD::SETCC.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21154 of file AArch64ISelLowering.cpp.
References SetCCInfo::AArch64, llvm::ISD::ADD, assert(), AArch64SetCCInfo::CC, GenericSetCCInfo::CC, AArch64SetCCInfo::Cmp, DL, SetCCInfo::Generic, getAArch64Cmp(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::ISD::getSetCCInverse(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), SetCCInfoAndKind::Info, SetCCInfoAndKind::IsAArch64, isSetCCOrZExtSetCC(), LHS, GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, RHS, SDValue(), and std::swap().
Referenced by performAddSubLongCombine().
|
static |
Definition at line 26057 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::cast(), Cond, DL, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSignedConstant(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::isa(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::APInt::isOne(), llvm::isOneConstant(), llvm::EVT::isScalarInteger(), LHS, N, performOrXorChainCombine(), RHS, SDValue(), llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, tryToWidenSetCCOperands(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().
|
static |
Definition at line 26214 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::cast(), Cond, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ISD::INSERT_SUBVECTOR, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), isAllActivePredicate(), llvm::isNullConstant(), isSignExtInReg(), isZerosVector(), LHS, N, performSetCCPunpkCombine(), RHS, SDValue(), llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, and llvm::ISD::UNDEF.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 26164 of file AArch64ISelLowering.cpp.
References llvm::cast(), Cond, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), isZerosVector(), LHS, N, RHS, SDValue(), llvm::ISD::SETNE, and llvm::ISD::SIGN_EXTEND.
Referenced by performSetccMergeZeroCombine().
|
static |
If the operand is a bitwise AND with a constant RHS, and the shift has a constant RHS and is the only use, we can pull it out of the shift, i.e.
(shl (and X, C1), C2) -> (and (shl X, C2), (shl C1, C2))
We prefer this canonical form to match existing isel patterns.
Definition at line 27617 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::hasOneUse(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, SDValue(), llvm::ISD::SETCC, llvm::ISD::SHL, llvm::ISD::SUB, and X.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().
|
static |
Definition at line 26981 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, EnableCombineMGatherIntrinsics, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), llvm::isOneConstant(), N, Opc, SDValue(), and llvm::ISD::SIGN_EXTEND_INREG.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23034 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::SDNode::getValueType(), isCheapToExtend(), llvm::EVT::isInteger(), N, SDValue(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by performExtendCombine().
|
static |
Definition at line 21710 of file AArch64ISelLowering.cpp.
References N, SDValue(), and trySQDMULHCombine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23637 of file AArch64ISelLowering.cpp.
References assert(), N, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23388 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, llvm::Data, DL, llvm::SelectionDAG::getNode(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::EVT::isFloatingPoint(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23414 of file AArch64ISelLowering.cpp.
References llvm::cast(), llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::SelectionDAG::getMaskedStore(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isFloatingPoint(), N, and llvm::ISD::UNINDEXED.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21859 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::Add, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::M1(), llvm::ISD::MUL, N, SDValue(), llvm::ISD::SUB, llvm::Sub, and X.
Referenced by performAddSubCombine().
|
static |
Definition at line 25443 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, assert(), DL, llvm::dyn_cast(), llvm::AArch64CC::EQ, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SDNode::getVTList(), llvm::AArch64CC::HI, llvm::APInt::isMask(), llvm::APInt::isPowerOf2(), llvm::AArch64CC::LO, N, llvm::AArch64CC::NE, and SDValue().
Referenced by performCONDCombine().
|
static |
Definition at line 24057 of file AArch64ISelLowering.cpp.
References llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), N, SDValue(), and llvm::ISD::SIGN_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20098 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::CallingConv::C, llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::dyn_cast(), EnableCombineMGatherIntrinsics, llvm::ISD::EXTLOAD, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), isAllActivePredicate(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isConstantSplatVectorMaskForType(), N, Opc, SDValue(), llvm::ISD::SPLAT_VECTOR, and llvm::ISD::ZEXTLOAD.
Referenced by performANDCombine().
|
static |
Definition at line 21895 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::cast(), convertFromScalableVector(), convertToScalableVector(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isScalableVector(), N, SDValue(), and llvm::ISD::SUB.
Referenced by performAddSubCombine().
|
static |
Definition at line 26354 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getTestBitOperand(), N, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21718 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::SDValue::hasOneUse(), llvm::EVT::is64BitVector(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isScalableVector(), N, SDValue(), and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19064 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), performUADDVAddCombine(), SDValue(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by performUADDVAddCombine(), and performUADDVCombine().
|
static |
Definition at line 19151 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, assert(), llvm::SelectionDAG::computeKnownBits(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorNumElements(), llvm::KnownBits::isZero(), N, performUADDVAddCombine(), performUADDVZextCombine(), and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19113 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, assert(), Concat, llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SDValue::getValueType(), llvm_unreachable, SDValue(), llvm::MVT::SimpleTy, and llvm::ISD::ZERO_EXTEND.
Referenced by performUADDVCombine().
|
static |
Definition at line 23647 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), llvm::EVT::changeVectorElementType(), DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedLoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::MaskedLoadSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SDValue::getNode(), llvm::getNumElementsFromSVEPredPattern(), llvm::MaskedLoadSDNode::getOffset(), llvm::MaskedLoadSDNode::getPassThru(), getPTrue(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::EVT::getVectorElementType(), llvm::SDNode::isUndef(), llvm::MaskedLoadStoreSDNode::isUnindexed(), isZerosVector(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), SDValue(), llvm::ISD::SEXTLOAD, and llvm::ISD::ZEXTLOAD.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23790 of file AArch64ISelLowering.cpp.
References Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorMinNumElements(), isHalvingTruncateAndConcatOfLegalIntScalableType(), llvm::DataLayout::isLittleEndian(), isNVCastToHalfWidthElements(), llvm::EVT::isSimple(), llvm_unreachable, N, llvm::peekThroughBitcasts(), SDValue(), llvm::MVT::SimpleTy, llvm::ISD::TRUNCATE, tryCombineExtendRShTrunc(), trySimplifySrlAddToRshrnb(), llvm::ISD::UNDEF, and X.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18939 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, B(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), I, llvm::EVT::isScalableVT(), llvm::ISD::MUL, N, performVecReduceAddCombineWithUADDLP(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::ISD::SIGN_EXTEND, std::swap(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18769 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ABS, llvm::ISD::ADD, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), N, SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, and llvm::ISD::ZERO_EXTEND.
Referenced by performVecReduceAddCombine().
|
static |
Definition at line 26042 of file AArch64ISelLowering.cpp.
References DL, llvm::SDValue::getValueType(), getVectorBitwiseReduce(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isPow2VectorType(), N, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19757 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::isVector(), N, SDValue(), and llvm::ISD::SETCC.
Referenced by performIntToFpCombine().
|
static |
Definition at line 27534 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorMinNumElements(), I, llvm::ISD::INTRINSIC_W_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isNormalMaskedLoad(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), isZerosVector(), N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19470 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getContext(), llvm::EVT::getHalfSizedIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::EVT::isFixedLengthVector(), llvm::ISD::MUL, N, S1, SDValue(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by performAddSubCombine(), and performMulCombine().
|
static |
Optimize a vector shift instruction and its operand if shifted out bits are not used.
Definition at line 24025 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::APInt::getLowBitsSet(), N, SDValue(), and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 26423 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ArrayRef(), llvm::cast(), llvm::EVT::changeVectorElementTypeToInteger(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::ElementCount::getFixed(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), llvm::is_contained(), isAllActivePredicate(), isAllInactivePredicate(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::EVT::isFloatingPoint(), isMergePassthruOpcode(), llvm::APInt::isOne(), llvm::EVT::isSimple(), llvm::SDValue::isUndef(), N, llvm::SDNode::op_values(), llvm::ISD::OR, SDValue(), llvm::ISD::SETCC, llvm::ISD::SETGT, llvm::ISD::SRA, trySwapVSelectOperands(), and llvm::ISD::VSELECT.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19173 of file AArch64ISelLowering.cpp.
References foldVectorXorShiftIntoCmp(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, and SDValue().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23076 of file AArch64ISelLowering.cpp.
References llvm::all_of(), llvm::ISD::AND, DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::ShuffleVectorInst::isDeInterleaveMaskOfFactor(), N, SDValue(), llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.
Referenced by performExtendCombine().
|
static |
Definition at line 23136 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::ISD::isConstantSplatVector(), N, SDValue(), and llvm::ISD::ZERO_EXTEND.
Referenced by performExtendCombine().
|
static |
Definition at line 25749 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, Check, llvm::dyn_cast(), llvm::AArch64CC::EQ, FlagsVT, llvm::AArch64CC::GE, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getSwappedCondition(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::AArch64CC::GT, llvm::SDValue::hasOneUse(), llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, N, llvm::AArch64CC::NE, llvm::SelectionDAG::ReplaceAllUsesWith(), SDValue(), llvm::ISD::SUB, std::swap(), X, and Y.
Referenced by performCSELCombine().
SDValue ReconstructShuffleWithRuntimeMask | ( | SDValue | Op, |
SelectionDAG & | DAG ) |
Definition at line 13350 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BUILD_VECTOR, llvm::cast(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isa(), llvm::EVT::isScalableVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and SDValue().
|
static |
Definition at line 13773 of file AArch64ISelLowering.cpp.
References assert(), llvm::sampleprof::Base, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::isa(), SDValue(), llvm::ISD::TRUNCATE, X, and Y.
Definition at line 27200 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isConstantSplatVectorAllZeros(), isLanes1toNKnownZero(), llvm::isNullConstant(), N, and SDValue().
Referenced by performInsertVectorEltCombine().
|
static |
Definition at line 28272 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::dyn_cast(), E(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::EVT::is256BitVector(), llvm::EVT::isFloatingPoint(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), Results, llvm::SelectionDAG::SplitVector(), and X.
|
static |
Definition at line 28577 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::ISD::BUILD_PAIR, llvm::cast(), DL, llvm::SelectionDAG::getAllOnesConstant(), getAtomicLoad128Opcode(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::DataLayout::isBigEndian(), llvm::Lo, N, Ptr, Results, SDValue(), llvm::SelectionDAG::setNodeMemRefs(), llvm::SelectionDAG::SplitScalar(), std::swap(), and llvm::ISD::XOR.
|
static |
Definition at line 28177 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), I, llvm::EVT::isVector(), N, and Results.
|
static |
Definition at line 28414 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::Acquire, llvm::AcquireRelease, assert(), llvm::ISD::BUILD_PAIR, llvm::cast(), createGPRPairNode(), DL, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::DataLayout::isBigEndian(), llvm_unreachable, llvm::Lo, llvm::Monotonic, N, llvm::Release, Results, SDValue(), llvm::SequentiallyConsistent, llvm::SelectionDAG::setNodeMemRefs(), llvm::SelectionDAG::SplitScalar(), and std::swap().
|
static |
Definition at line 28320 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::Hi, llvm::Lo, N, Results, and llvm::SelectionDAG::SplitVectorOperand().
|
static |
Replace a splat of a scalar to a vector store by scalar stores of the scalar value.
The load store optimizer pass will merge them to store pair stores. This has better performance than a splat of the scalar followed by a split vector store. Even if the stores are not merged it is four stores vs a dup, followed by an ext.b and two stores.
Definition at line 23513 of file AArch64ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), I, llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::isFloatingPoint(), llvm::StoreSDNode::isTruncatingStore(), SDValue(), and splitStoreSplat().
Referenced by splitStores().
|
static |
Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.
The load store optimizer pass will merge them to store pair stores. This should be better than a movi to create the vector zero followed by a vector store if the zero constant is not re-used, since one instructions and one register live range will be removed.
For example, the final generated code should be:
stp xzr, xzr, [x0]
instead of:
movi v0.2d, #0 str q0, [x0]
Definition at line 23446 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, DL, llvm::StoreSDNode::getBasePtr(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), I, llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::isNullConstant(), llvm::isNullFPConstant(), llvm::EVT::isScalableVector(), llvm::StoreSDNode::isTruncatingStore(), llvm::Offset, SDValue(), and splitStoreSplat().
Referenced by splitStores().
|
static |
Definition at line 14770 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::BuildVectorSDNode::isConstantSplat(), and llvm::APInt::zextOrTrunc().
Referenced by ConstantBuildVector(), and performANDCombine().
|
static |
Definition at line 29044 of file AArch64ISelLowering.cpp.
References llvm::Type::BFloatTyID, llvm::Type::DoubleTyID, llvm::Type::FloatTyID, llvm::Type::getScalarType(), llvm::Value::getType(), llvm::Type::getTypeID(), llvm::Type::HalfTyID, llvm::AtomicRMWInst::isFloatingPointOperation(), and llvm_unreachable.
Referenced by llvm::AArch64TargetLowering::shouldExpandAtomicRMWInIR().
return SDValue | ( | ) |
Referenced by AddCombineBUILD_VECTORToVPADDL(), AddCombineTo64bitMLAL(), AddCombineTo64BitSMLAL16(), AddCombineTo64bitUMAAL(), AddCombineToVPADD(), AddCombineVUZPToVPADDL(), llvm::InstrEmitter::AddDbgValueLocationOps(), llvm::AMDGPUTargetLowering::addTokenForArgument(), adjustBitcastSrcVectorSSE1(), adjustForFNeg(), adjustForLTGFR(), adjustForSubtraction(), adjustForTestUnderMask(), adjustSubwordCmp(), llvm::RISCVDAGToDAGISel::areOffsetsWithinAlignment(), AVRDAGToDAGISel::select< AVRISD::CALL >(), AVRDAGToDAGISel::select< ISD::BRIND >(), AVRDAGToDAGISel::select< ISD::LOAD >(), AVRDAGToDAGISel::select< ISD::STORE >(), BuildExactSDIV(), BuildExactUDIV(), buildFromShuffleMostly(), llvm::TargetLowering::buildLegalVectorShuffle(), buildRegSequence16(), llvm::SITargetLowering::buildRSRC(), llvm::TargetLowering::BuildSDIV(), llvm::PPCTargetLowering::BuildSDIVPow2(), llvm::TargetLowering::BuildSDIVPow2(), buildSMovImm32(), llvm::TargetLowering::BuildSREMPow2(), llvm::TargetLowering::BuildUDIV(), canonicalizeBitSelect(), canonicalizeLaneShuffleWithRepeatedOps(), canonicalizePRMTInput(), canonicalizeShuffleMaskWithHorizOp(), canonicalizeShuffleWithOp(), llvm::AArch64TargetLowering::changeStreamingMode(), checkBoolTestSetCCCombine(), checkIntrinsicImmArg(), checkSignTestSetCCCombine(), combineADC(), combineAddOfBooleanXor(), combineAddOfPMADDWD(), combineAddOrSubToADCOrSBB(), combineAddOrSubToADCOrSBB(), combineADDRSPACECAST(), combineADDToADDZE(), combineADDToMAT_PCREL_ADDR(), combineADDX(), combineAnd(), combineAndLoadToBZHI(), combineAndMaskToShift(), combineAndNotIntoANDNP(), combineAndNotOrIntoAndNotAnd(), combineAndnp(), combineAndOrForCcmpCtest(), CombineANDShift(), combineAndShuffleNot(), combineAndXorSubWithBMI(), combineArithReduction(), combineAVG(), combineAVX512SetCCToKMOV(), combineBallotPattern(), CombineBaseUpdate(), combineBasicSADPattern(), combineBEXTR(), combineBinOpOfExtractToReduceTree(), combineBinOpOfZExt(), combineBinOpToReduce(), combineBitcast(), combineBitcastToBoolVector(), combineBitcastvxi1(), combineBitOpWithMOVMSK(), combineBitOpWithPACK(), combineBitOpWithShift(), combineBITREVERSE(), combineBlendOfPermutes(), combineBMILogicOp(), combineBrCond(), combineBROADCAST_LOAD(), combineBT(), combineBVOfConsecutiveLoads(), combineBVOfVecSExt(), combineBVZEXTLOAD(), combineCarryDiamond(), combineCarryThroughADD(), combineCarryThroughADD(), combineCastedMaskArithmetic(), combineCMov(), combineCMP(), combineCommutableSHUFP(), combineCompareEqual(), combineCONCAT_VECTORS(), combineConcatVectorOfCasts(), combineConcatVectorOfConcatVectors(), combineConcatVectorOfExtracts(), combineConcatVectorOfScalars(), combineConcatVectorOfShuffleAndItsOperands(), combineConcatVectorOfSplats(), combineConcatVectorOps(), combineConstantPoolLoads(), combineCVTP2I_CVTTP2I(), combineCVTPH2PS(), combineDeMorganOfBoolean(), combineEXTEND_VECTOR_INREG(), combineEXTRACT_SUBVECTOR(), combineExtractFromVectorLoad(), combineExtractVectorElt(), combineExtractWithShuffle(), combineExtSetcc(), combineFaddCFmul(), combineFaddFsub(), combineFAndFNotToFAndn(), combineFMA(), combineFMADDSUB(), combineFMinFMax(), llvm::AMDGPUTargetLowering::combineFMinMaxLegacy(), llvm::AMDGPUTargetLowering::combineFMinMaxLegacyImpl(), combineFMinNumFMaxNum(), combineFneg(), combineFP16_TO_FP(), combineFP_EXTEND(), combineFP_ROUND(), combineFP_TO_xINT_SAT(), combineFPToSInt(), combineGatherScatter(), combineHorizOpWithShuffle(), combinei64TruncSrlConstant(), combineINSERT_SUBVECTOR(), combineINTRINSIC_VOID(), combineINTRINSIC_W_CHAIN(), combineINTRINSIC_WO_CHAIN(), combineKSHIFT(), combineLoad(), combineLogicBlendIntoConditionalNegate(), combineLogicBlendIntoPBLENDV(), combineLRINT_LLRINT(), combineM68kBrCond(), combineM68kSetCC(), combineMADConstOne(), combineMaskedLoad(), combineMaskedLoadConstantMask(), combineMaskedStore(), combineMinMaxReduction(), combineMinNumMaxNumImpl(), combineMOVDQ2Q(), combineMOVMSK(), combineMul(), combineMulSelectConstOne(), combineMulSpecial(), combineMulToPMADDWD(), combineMulToPMULDQ(), combineMulWide(), combineOp_VLToVWOp_VL(), combineOr(), combineOrAndToBitfieldInsert(), combineOrCmpEqZeroToCtlzSrl(), combineOrOfCZERO(), combineOrToBitfieldInsert(), combineOrXorWithSETCC(), combinePackingMovIntoStore(), combinePDEP(), combinePMULDQ(), combinePMULH(), combinePredicateReduction(), combinePRMT(), combineProxyReg(), combinePTESTCC(), combineRedundantDWordShuffle(), combineSBB(), combineSCALAR_TO_VECTOR(), combineScalarAndWithMaskSetcc(), combineSelect(), llvm::VETargetLowering::combineSelect(), combineSelectAndUse(), combineSelectAndUse(), combineSelectAndUse(), combineSelectAndUseCommutative(), combineSelectAndUseCommutative(), combineSelectAndUseCommutative(), combineSelectAsExtAnd(), llvm::VETargetLowering::combineSelectCC(), combineSelectOfTwoConstants(), combineSelectToBinOp(), combineSetCC(), combineSetCCAtomicArith(), combineSetCCCCR(), combineSetCCMOVMSK(), combineSext(), combineSextInRegCmov(), combineShiftAnd1ToBitTest(), combineShiftLeft(), combineShiftOfShiftedLogic(), combineShiftRightArithmetic(), combineShiftRightLogical(), combineShiftToAVG(), combineShiftToMULH(), combineShiftToPMULH(), combineShlAddIAdd(), combineShlAddIAddImpl(), combineShuffle(), combineShuffleOfBitcast(), combineShuffleOfScalars(), combineShuffleOfSplatVal(), combineShuffleToAddSubOrFMAddSub(), combineShuffleToAnyExtendVectorInreg(), combineShuffleToFMAddSub(), combineShuffleToZeroExtendVectorInReg(), combineSignExtendInReg(), combineSIntToFP(), combineSTORE(), combineStore(), combineSubABS(), combineSubOfBoolean(), combineSubSetcc(), combineSubShiftToOrcB(), combineSUBX(), combineSVEPrefetchVecBaseImmOff(), combineTargetShuffle(), combineTESTP(), combineToConsecutiveLoads(), combineToExtendBoolVectorInReg(), combineToExtendCMOV(), combineToFPTruncExtElt(), combineToHorizontalAddSub(), combineToVCPOP(), combineToVWMACC(), llvm::VETargetLowering::combineTRUNCATE(), combineTruncate(), combineTruncatedArithmetic(), combineTruncateWithSat(), combineTruncationShuffle(), combineTruncOfSraSext(), combineTruncSelectToSMaxUSat(), combineTruncToVnclip(), combineUADDO_CARRYDiamond(), combineUIntToFP(), combineUnpackingMovIntoLoad(), combineVectorCompare(), combineVectorCompareAndMaskUnaryOp(), combineVectorHADDSUB(), combineVectorInsert(), combineVectorMulToSraBitcast(), combineVectorPack(), combineVectorShiftImm(), combineVectorShiftVar(), combineVectorSizedSetCCEquality(), combineVectorSizedSetCCEquality(), combineVectorSizedSetCCEquality(), combineVEXTRACT_STORE(), combineVFMADD_VLWithVFNEG_VL(), CombineVLDDUP(), CombineVMOVDRRCandidateWithVecOp(), combineVPDPBUSDPattern(), combineVPMADD(), combineVPMADD52LH(), combineVqdotAccum(), combineVSelectToBLENDV(), combineVSelectWithAllOnesOrZeros(), combineVTRUNC(), combineVWADDSUBWSelect(), combineX86AddSub(), combineX86CloadCstore(), combineX86GatherScatter(), combineX86INT_TO_FP(), combineX86SetCC(), combineX86ShuffleChain(), combineX86ShuffleChainWithExtract(), combineX86ShufflesConstants(), combineX86ShufflesRecursively(), combineX86SubCmpForFlags(), combineXor(), combineXorSubCTLZ(), combineXorToBitfieldInsert(), combineZext(), commuteSelect(), ConstantBuildVector(), constructRetValue(), convertFPToInt(), convertFromF16(), convertIntLogicToFPLogic(), convertMergedOpToPredOp(), ConvertSelectToConcatVector(), convertShiftLeftToScale(), llvm::SITargetLowering::copyToM0(), createGPRPairNode(), createGPRPairNode2xi32(), llvm::TargetLowering::createSelectForFMINNUM_FMAXNUM(), createVariablePermute(), llvm::TargetLowering::CTTZTableLookup(), DAGCombineAddc(), detectPMADDUBSW(), detectSSatPattern(), detectSSatSPattern(), detectSSatUPattern(), detectUSatPattern(), detectUSatUPattern(), llvm::HexagonDAGToDAGISel::DetectUseSxtw(), llvm::SDNode::DropOperands(), eliminateFPCastPair(), EltsFromConsecutiveLoads(), EmitAVX512Test(), emitCmp(), emitConjunction(), emitConstantSizeRepmov(), emitConstantSizeRepstos(), llvm::InstrEmitter::EmitDbgInstrRef(), emitIntrinsicWithCCAndChain(), emitLockedStackOp(), llvm::AArch64SelectionDAGInfo::EmitMOPS(), llvm::ARMSelectionDAGInfo::EmitSpecializedLibcall(), llvm::X86TargetLowering::emitStackGuardXorFP(), llvm::AArch64SelectionDAGInfo::EmitStreamingCompatibleMemLibCall(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForMemchr(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForMemcmp(), llvm::AArch64SelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::ARMSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::BPFSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::HexagonSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::LanaiSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForMemcpy(), llvm::SystemZSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::WebAssemblySelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::X86SelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::XCoreSelectionDAGInfo::EmitTargetCodeForMemcpy(), llvm::AArch64SelectionDAGInfo::EmitTargetCodeForMemmove(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForMemmove(), llvm::AArch64SelectionDAGInfo::EmitTargetCodeForMemset(), llvm::ARMSelectionDAGInfo::EmitTargetCodeForMemset(), llvm::RISCVSelectionDAGInfo::EmitTargetCodeForMemset(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForMemset(), llvm::SystemZSelectionDAGInfo::EmitTargetCodeForMemset(), llvm::WebAssemblySelectionDAGInfo::EmitTargetCodeForMemset(), llvm::X86SelectionDAGInfo::EmitTargetCodeForMemset(), llvm::AArch64SelectionDAGInfo::EmitTargetCodeForSetTag(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForSetTag(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForStrcmp(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForStrcpy(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForStrlen(), llvm::SelectionDAGTargetInfo::EmitTargetCodeForStrnlen(), EmitTest(), emitVectorComparison(), Expand64BitShift(), llvm::TargetLowering::expandABS(), expandBitCastI128ToF128(), llvm::TargetLowering::expandBSWAP(), llvm::TargetLowering::expandCTLZ(), llvm::TargetLowering::expandCTPOP(), llvm::TargetLowering::expandCTTZ(), llvm::TargetLowering::expandFixedPointDiv(), llvm::TargetLowering::expandFixedPointMul(), llvm::TargetLowering::expandFMINNUM_FMAXNUM(), llvm::TargetLowering::expandFP_ROUND(), llvm::TargetLowering::expandFP_TO_INT_SAT(), llvm::TargetLowering::expandFP_TO_SINT(), llvm::TargetLowering::expandFP_TO_UINT(), expandFSH64(), llvm::TargetLowering::expandFunnelShift(), expandIntrinsicWChainHelper(), expandMul(), llvm::TargetLowering::expandMUL_LOHI(), llvm::SelectionDAG::expandMultipleResultFPLibCall(), expandMulToAddOrSubOfShl(), llvm::TargetLowering::expandROT(), llvm::TargetLowering::expandUADDSUBO(), llvm::TargetLowering::expandUINT_TO_FP(), llvm::TargetLowering::expandVectorNaryOpBySplitting(), llvm::TargetLowering::expandVPBITREVERSE(), llvm::TargetLowering::expandVPBSWAP(), llvm::TargetLowering::expandVPCTPOP(), expandVPFunnelShift(), llvm::PPCTargetLowering::expandVSXLoadForLE(), llvm::PPCTargetLowering::expandVSXStoreForLE(), extractBooleanFlip(), extractShiftForRotate(), llvm::HexagonDAGToDAGISel::FastFDiv(), llvm::HexagonDAGToDAGISel::FDiv(), FindBFIToCombineWith(), findMemSDNode(), findMUL_LOHI(), findVSplat(), FixupMMXIntrinsicTypes(), foldADCToCINC(), foldAddSubBoolOfMaskedVal(), foldAddSubMasked1(), foldAddSubOfSignBit(), foldAndOrOfSETCC(), foldAndToUsubsat(), foldBinOpIntoSelectIfProfitable(), foldBinOpIntoSelectIfProfitable(), foldBitOrderCrossLogicOp(), foldBoolSelectToLogic(), FoldBUILD_VECTOR(), foldCONCAT_VECTORS(), foldConcatVector(), llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::FoldConstantBuildVector(), llvm::SelectionDAG::foldConstantFPMath(), foldCSELOfCSEL(), foldCSELofCTTZ(), foldCSELofLASTB(), foldExtendedSignBitTest(), foldExtendVectorInregToExtendOfSubvector(), foldExtractSubvectorFromShuffleVector(), foldFPToIntToFP(), llvm::AMDGPUTargetLowering::foldFreeOpFromSelect(), FoldIntToFPToInt(), foldLogicOfShifts(), foldLogicTreeOfShifts(), foldMaskedMerge(), foldOverflowCheck(), foldReduceOperandViaVQDOT(), foldRemainderIdiom(), foldSelectOfConstantsUsingSra(), foldSelectOfCTTZOrCTLZ(), foldSelectWithIdentityConstant(), llvm::SelectionDAG::FoldSetCC(), foldSetCCWithFunnelShift(), foldSetCCWithRotate(), foldShuffleOfConcatUndefs(), FoldSTEP_VECTOR(), foldSubCtlzNot(), llvm::SelectionDAG::FoldSymbolOffset(), foldToMaskedStore(), foldToSaturated(), foldVectorXorShiftIntoCmp(), foldVectorXorShiftIntoCmp(), foldVSelectToSignBitSplatMask(), foldXor1SetCC(), foldXorTruncShiftIntoCmp(), formSplatFromShuffles(), GenerateFixedLengthSVETBL(), llvm::PPC::get_VSPLTI_elt(), getAddOneOp(), llvm::SelectionDAG::getAddrSpaceCast(), llvm::getAnnotatedNodeAVL(), getAsCarry(), llvm::SelectionDAG::getAssertAlign(), llvm::SelectionDAG::getAtomic(), getAVX2GatherNode(), getAVX512Node(), getAVX512TruncNode(), llvm::SelectionDAG::getBasicBlock(), llvm::getBitwiseNotOperand(), llvm::SelectionDAG::getBlockAddress(), getBMIMatchingOp(), getBROADCAST_LOAD(), getBT(), llvm::TargetLowering::getCheaperOrNeutralNegatedExpression(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getConstantPool(), llvm::SelectionDAG::getConstantPool(), llvm::RegsForValue::getCopyFromRegs(), getDataClassTest(), llvm::SelectionDAG::getEntryNode(), getEstimate(), GetExtendHigh(), llvm::SelectionDAG::getExternalSymbol(), getFlagsOfCmpZeroFori1(), llvm::SelectionDAG::getFrameIndex(), getGatherNode(), llvm::getGatherScatterIndex(), llvm::getGatherScatterScale(), llvm::SelectionDAG::getGatherVP(), llvm::SelectionDAG::getGetFPEnv(), llvm::SelectionDAG::getGlobalAddress(), llvm::SelectionDAG::getIndexedStoreVP(), getIndexFromUnindexedLoad(), getInputChainForNode(), getIntrinsicCmp(), getInvertedVectorForFMA(), llvm::SelectionDAG::getJumpTable(), getKnownUndefForVectorBinop(), llvm::SelectionDAG::getLabelNode(), getLeftShift(), llvm::SelectionDAG::getLifetimeNode(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getLoadFFVP(), getLoadStackGuard(), llvm::getLoadStoreStride(), llvm::SelectionDAG::getLoadVP(), llvm::StatepointLoweringState::getLocation(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getMaskedGather(), llvm::SelectionDAG::getMaskedHistogram(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMaskedScatter(), llvm::SelectionDAG::getMaskedStore(), llvm::SelectionDAG::getMCSymbol(), llvm::SelectionDAG::getMDNode(), getMemcpyLoadsAndStores(), llvm::SelectionDAG::getMemIntrinsicNode(), getMemmoveLoadsAndStores(), llvm::getMemoryPtr(), getMemsetStores(), getMemsetStringVal(), llvm::AMDGPUTargetLowering::getNegatedExpression(), llvm::PPCTargetLowering::getNegatedExpression(), llvm::TargetLowering::getNegatedExpression(), llvm::SelectionDAG::getNeutralElement(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNode(), llvm::getNodeAVL(), llvm::getNodeChain(), llvm::getNodeMask(), llvm::getNodePassthru(), getNullFPConstForNullVal(), getPrefetchNode(), llvm::SelectionDAG::getPseudoProbeNode(), llvm::AMDGPUTargetLowering::getRecipEstimate(), llvm::LoongArchTargetLowering::getRecipEstimate(), llvm::TargetLowering::getRecipEstimate(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getRegisterMask(), llvm::BuildVectorSDNode::getRepeatedSequence(), getScalarValueForVectorElement(), getScatterNode(), llvm::SelectionDAG::getScatterVP(), llvm::SelectionDAG::getSetFPEnv(), getShuffleScalarElt(), getSingleShuffleSrc(), llvm::SelectionDAG::getSplatSourceVector(), getSplatValue(), llvm::BuildVectorSDNode::getSplatValue(), llvm::SelectionDAG::getSplatValue(), getSplitVectorSrc(), llvm::AMDGPUTargetLowering::getSqrtEstimate(), llvm::LoongArchTargetLowering::getSqrtEstimate(), llvm::NVPTXTargetLowering::getSqrtEstimate(), llvm::TargetLowering::getSqrtEstimate(), llvm::SelectionDAG::getSrcValue(), llvm::SelectionDAG::getStackArgumentTokenFactor(), llvm::SelectionDAG::getStore(), llvm::getStoredValue(), llvm::SelectionDAG::getStoreVP(), llvm::SelectionDAG::getStrictFPExtendOrRound(), llvm::SelectionDAG::getStridedLoadVP(), llvm::SelectionDAG::getStridedStoreVP(), getSubVectorSrc(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SelectionDAG::getTargetInsertSubreg(), GetTLSADDR(), getTruncatedUSUBSAT(), llvm::SelectionDAG::getTruncStoreVP(), llvm::SelectionDAG::getTruncStridedStoreVP(), getUsefulBitsForUse(), llvm::SelectionDAGBuilder::getValueImpl(), llvm::SelectionDAG::getValueType(), getVectorBitwiseReduce(), llvm::SelectionDAG::getVectorShuffle(), getVLOperand(), HandleMergeInputChains(), if(), incDecVectorConstant(), INITIALIZE_PASS(), InvertCarryFlag(), IsCMPZCSINC(), isFNEG(), llvm::TargetLowering::isGAPlusOffset(), isHorizontalBinOp(), llvm::GCNTTIImpl::isInlineAsmSourceOfDivergence(), IsNOT(), isNVCastToHalfWidthElements(), isOnlyUsedByStores(), IsOperandAMemoryOperand(), isSaturatingMinMax(), isScalarToVec(), IsSingleInstrConstant(), isTargetShuffleEquivalent(), isUpperSubvectorUndef(), isVMOVModifiedImm(), isWorthFoldingIntoOrrWithShift(), isWorthFoldingIntoRegRegScale(), llvm::RISCVTargetLowering::joinRegisterPartsIntoValue(), llvm::SystemZTargetLowering::joinRegisterPartsIntoValue(), llvm::TargetLowering::joinRegisterPartsIntoValue(), llvm::TargetLowering::LegalizeSetCCCondCode(), legalizeSVEGatherPrefetchOffsVec(), llvm::SITargetLowering::legalizeTargetIndependentNode(), lower128BitShuffle(), lower1BitShuffle(), lower1BitShuffleAsKSHIFTR(), lower256BitShuffle(), LowerABD(), LowerABS(), LowerADDSAT_SUBSAT(), LowerADDSUBO_CARRY(), lowerADDSUBO_CARRY(), LowerADDSUBSAT(), LowerAndToBT(), LowerAndToBTST(), llvm::TargetLowering::LowerAsmOutputForConstraint(), llvm::X86TargetLowering::LowerAsmOutputForConstraint(), LowerAsSplatVectorLoad(), llvm::SelectionDAGBuilder::LowerAsSTATEPOINT(), llvm::VETargetLowering::lowerATOMIC_FENCE(), LowerATOMIC_LOAD_STORE(), LowerAtomicLoadStore(), LowerAVG(), lowerBALLOTIntrinsic(), llvm::HexagonTargetLowering::LowerBITCAST(), LowerBITCAST(), lowerBitreverseShuffle(), LowerBRCOND(), llvm::HexagonTargetLowering::LowerBUILD_VECTOR(), llvm::VETargetLowering::lowerBUILD_VECTOR(), lowerBUILD_VECTOR(), LowerBUILD_VECTOR_i1(), lowerBUILD_VECTORAsBroadCastLoad(), LowerBUILD_VECTORAsVariablePermute(), LowerBUILD_VECTORToVIDUP(), lowerBuildVectorAsBlend(), lowerBuildVectorAsBroadcast(), lowerBuildVectorOfConstants(), LowerBuildVectorOfFPExt(), LowerBuildVectorOfFPTrunc(), lowerBuildVectorToBitOp(), LowerBuildVectorv16i8(), LowerBuildVectorv4x32(), LowerBuildVectorv8i16(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaPacking(), lowerBuildVectorViaVID(), llvm::HexagonTargetLowering::LowerCall(), llvm::NVPTXTargetLowering::LowerCall(), llvm::SITargetLowering::LowerCall(), llvm::VETargetLowering::LowerCall(), LowerCallResult(), lowerCallResult(), llvm::TargetLowering::LowerCallTo(), LowerClusterLaunchControlQueryCancel(), llvm::TargetLowering::lowerCmpEqZeroToCtlzSrl(), llvm::HexagonTargetLowering::LowerCONCAT_VECTORS(), lowerConstant(), LowerConvertLow(), LowerCTPOP(), LowerCTTZ(), lowerDisjointIndicesShuffle(), llvm::AMDGPUTargetLowering::LowerDIVREM24(), lowerDSPIntr(), llvm::NVPTXTargetLowering::LowerDYNAMIC_STACKALLOC(), llvm::SITargetLowering::LowerDYNAMIC_STACKALLOC(), llvm::VETargetLowering::lowerDYNAMIC_STACKALLOC(), llvm::TargetLowering::lowerEHPadEntry(), LowerEXTEND_VECTOR_INREG(), LowerEXTRACT_VECTOR_ELT(), llvm::VETargetLowering::lowerEXTRACT_VECTOR_ELT(), LowerEXTRACT_VECTOR_ELT_SSE4(), LowerF128_FPEXTEND(), LowerF128_FPROUND(), LowerF128Load(), LowerF64Op(), llvm::HexagonTargetLowering::LowerFDIV(), llvm::AMDGPUTargetLowering::lowerFEXP(), LowerFLDEXP(), LowerFNEGorFABS(), llvm::VETargetLowering::LowerFormalArguments(), llvm::AMDGPUTargetLowering::LowerFP_TO_FP16(), llvm::AMDGPUTargetLowering::LowerFP_TO_INT(), LowerFP_TO_INT_SAT(), lowerFP_TO_INT_SAT(), LowerFP_TO_SINT(), lowerFP_TO_SINT_STORE(), LowerFP_TO_UINT(), lowerFPToIntToFP(), lowerFTRUNC_FCEIL_FFLOOR_FROUND(), LowerFunnelShift(), LowerFunnelShift(), llvm::AMDGPUTargetLowering::LowerGlobalAddress(), lowerI128ToGR128(), LowerI64IntToFP16(), LowerI64IntToFP_AVX512DQ(), llvm::VETargetLowering::lowerINSERT_VECTOR_ELT(), lowerINT_TO_FP_vXi64(), llvm::HexagonTargetLowering::LowerINTRINSIC_VOID(), LowerINTRINSIC_W_CHAIN(), llvm::HexagonTargetLowering::LowerINTRINSIC_WO_CHAIN(), llvm::SparcTargetLowering::LowerINTRINSIC_WO_CHAIN(), llvm::VETargetLowering::lowerINTRINSIC_WO_CHAIN(), lowerIntrinsicWChain(), lowerLaneOp(), llvm::HexagonTargetLowering::LowerLoad(), LowerLoad(), llvm::MipsTargetLowering::lowerLOAD(), lowerLoadF128(), lowerLoadI1(), lowerLoadVector(), LowerMGATHER(), LowerMINMAX(), LowerMSCATTER(), llvm::LanaiTargetLowering::LowerMUL(), LowerMUL(), llvm::AArch64TargetLowering::LowerOperation(), llvm::ARMTargetLowering::LowerOperation(), llvm::AVRTargetLowering::LowerOperation(), llvm::HexagonTargetLowering::LowerOperation(), llvm::LoongArchTargetLowering::LowerOperation(), llvm::MipsTargetLowering::LowerOperation(), llvm::NVPTXTargetLowering::LowerOperation(), llvm::PPCTargetLowering::LowerOperation(), llvm::R600TargetLowering::LowerOperation(), llvm::RISCVTargetLowering::LowerOperation(), llvm::SITargetLowering::LowerOperation(), llvm::SystemZTargetLowering::LowerOperationWrapper(), llvm::TargetLowering::LowerOperationWrapper(), LowerPARITY(), llvm::SITargetLowering::lowerPREFETCH(), LowerPtrAuthGlobalAddressStatically(), llvm::VETargetLowering::LowerReturn(), llvm::SparcTargetLowering::LowerReturn_32(), LowerRotate(), llvm::HexagonTargetLowering::LowerROTL(), LowerSaturatingConditional(), lowerSelectToBinOp(), LowerSELECTWithCmpZero(), llvm::HexagonTargetLowering::LowerSETCC(), LowerShift(), LowerShift(), LowerShiftByScalarImmediate(), LowerShiftByScalarVariable(), lowerShuffleAsBitBlend(), lowerShuffleAsBitMask(), lowerShuffleAsBitRotate(), lowerShuffleAsBlend(), lowerShuffleAsBlendAndPermute(), lowerShuffleAsBroadcast(), lowerShuffleAsByteRotate(), lowerShuffleAsByteRotateAndPermute(), lowerShuffleAsByteShiftMask(), lowerShuffleAsElementInsertion(), lowerShuffleAsInsertPS(), lowerShuffleAsLanePermuteAndPermute(), lowerShuffleAsLanePermuteAndRepeatedMask(), lowerShuffleAsPermuteAndUnpack(), lowerShuffleAsRepeatedMaskAndLanePermute(), lowerShuffleAsShift(), lowerShuffleAsSpecificExtension(), lowerShuffleAsTruncBroadcast(), lowerShuffleAsUNPCKAndPermute(), lowerShuffleAsVALIGN(), lowerShuffleAsVTRUNC(), lowerShuffleAsVTRUNCAndUnpack(), lowerShuffleAsZeroOrAnyExtend(), lowerShuffleOfExtractsAsVperm(), lowerShufflePairAsUNPCKAndPermute(), lowerShuffleViaVRegSplitting(), lowerShuffleWithEXPAND(), lowerShuffleWithPACK(), lowerShuffleWithPSHUFB(), lowerShuffleWithSHUFPD(), lowerShuffleWithSSE4A(), lowerShuffleWithUndefHalf(), lowerShuffleWithUNPCK(), lowerShuffleWithUNPCK256(), lowerShuffleWithVPMOV(), LowerSINT_TO_FP(), LowerSMELdrStr(), llvm::NVPTXTargetLowering::LowerSTACKSAVE(), LowerSTORE(), LowerSTORE(), llvm::HexagonTargetLowering::LowerStore(), LowerStore(), llvm::VETargetLowering::lowerSTORE(), lowerStoreI1(), lowerSTOREVector(), LowerSVEIntrinsicEXT(), lowerToAddSubOrFMAddSub(), LowerToHorizontalOp(), llvm::VETargetLowering::lowerToVVP(), LowerTruncate(), LowerTruncateToBTST(), LowerTruncateVecPack(), LowerTruncateVecPackWithSignBits(), llvm::HexagonTargetLowering::LowerUAddSubO(), LowerUINT_TO_FP(), lowerUINT_TO_FP_v2i32(), lowerUINT_TO_FP_vXi32(), lowerV16I8Shuffle(), lowerV2X128Shuffle(), lowerV4X128Shuffle(), LowerVecReduce(), LowerVecReduceF(), LowerVecReduceMinMax(), lowerVECTOR_COMPRESS(), llvm::HexagonTargetLowering::LowerVECTOR_SHIFT(), llvm::HexagonTargetLowering::LowerVECTOR_SHUFFLE(), LowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLE_ILVEV(), lowerVECTOR_SHUFFLE_ILVL(), lowerVECTOR_SHUFFLE_ILVOD(), lowerVECTOR_SHUFFLE_ILVR(), lowerVECTOR_SHUFFLE_PCKEV(), lowerVECTOR_SHUFFLE_PCKOD(), lowerVECTOR_SHUFFLE_SHF(), lowerVECTOR_SHUFFLE_VILVH(), lowerVECTOR_SHUFFLE_VILVL(), lowerVECTOR_SHUFFLE_VPACKEV(), lowerVECTOR_SHUFFLE_VPACKOD(), lowerVECTOR_SHUFFLE_VPICKEV(), lowerVECTOR_SHUFFLE_VPICKOD(), lowerVECTOR_SHUFFLE_VREPLVEI(), lowerVECTOR_SHUFFLE_VSHUF4I(), lowerVECTOR_SHUFFLE_XVILVH(), lowerVECTOR_SHUFFLE_XVILVL(), lowerVECTOR_SHUFFLE_XVPERM(), lowerVECTOR_SHUFFLE_XVPERMI(), lowerVECTOR_SHUFFLE_XVPICKEV(), lowerVECTOR_SHUFFLE_XVPICKOD(), lowerVECTOR_SHUFFLE_XVREPLVEI(), lowerVECTOR_SHUFFLE_XVSHUF(), lowerVECTOR_SHUFFLE_XVSHUF4I(), lowerVECTOR_SHUFFLEAsByteRotate(), lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(), lowerVECTOR_SHUFFLEAsRotate(), lowerVECTOR_SHUFFLEAsShift(), lowerVECTOR_SHUFFLEAsVRGatherVX(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(), LowerVECTOR_SHUFFLEUsingMovs(), LowerVECTOR_SHUFFLEUsingOneOff(), LowerVectorAllEqual(), LowerVectorCTPOP(), LowerVectorExtend(), lowerVectorIntrinsicScalars(), llvm::HexagonTargetLowering::LowerVSELECT(), lowerVSELECTtoVectorShuffle(), LowerVSETCC(), LowerVSETCC(), LowerVSETCCWithSUBUS(), llvm::VETargetLowering::lowerVVP_GATHER_SCATTER(), llvm::VETargetLowering::lowerVVP_LOAD_STORE(), lowerX86FPLogicOp(), LowerXALUO(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), matchBF16FPExtendLike(), matchBinaryPermuteShuffle(), llvm::SelectionDAG::matchBinOpReduction(), matchBSwapHWordOrAndAnd(), matchExtFromI32orI32(), llvm::AMDGPUDAGToDAGISel::matchLoadD16FromBuildVector(), matchLSNode(), matchMADConstOnePattern(), matchMergedBFX(), matchPERM(), matchPMADDWD(), matchPMADDWD_2(), matchSplatAsGather(), matchTruncateWithPACK(), MatchVectorAllEqualTest(), matchVPMADD52(), materializeVectorConstant(), MergeInputChains(), llvm::SelectionDAG::MorphNodeTo(), moveBelowOrigChain(), llvm::SelectionDAG::mutateStrictFPToFP(), narrowExtractedVectorBinOp(), narrowExtractedVectorLoad(), narrowExtractedVectorSelect(), narrowInsertExtractVectorBinOp(), narrowLoadToVZLoad(), narrowShuffle(), narrowVectorSelect(), llvm::DAGTypeLegalizer::NoteDeletion(), optimizeIncrementingWhile(), optimizeLogicalImm(), partitionShuffleOfConcats(), performActiveLaneMaskCombine(), PerformADDCombine(), performADDCombine(), performAddCombineForShiftedOperands(), performAddCombineSubShift(), PerformADDCombineWithOperands(), PerformADDCombineWithOperands(), performAddCSelIntoCSinc(), PerformAddcSubcCombine(), performAddDotCombine(), PerformADDECombine(), PerformAddeSubeCombine(), performAddSubIntoVectorOp(), performAddSubLongCombine(), performAddUADDVCombine(), PerformADDVecReduce(), PerformANDCombine(), performANDCombine(), performANDCombine(), performANDCombine(), performANDCombine(), performANDORCSELCombine(), performANDSETCCCombine(), performAnyAllCombine(), PerformARMBUILD_VECTORCombine(), llvm::AMDGPUTargetLowering::performAssertSZExtCombine(), PerformBFICombine(), llvm::SparcTargetLowering::PerformBITCASTCombine(), PerformBITCASTCombine(), performBITCASTCombine(), performBitcastCombine(), performBITREV_WCombine(), performBITREVERSECombine(), performBR_CCCombine(), llvm::ARMTargetLowering::PerformBRCONDCombine(), performBRCONDCombine(), performBSPExpandForSVE(), PerformBUILD_VECTORCombine(), PerformBUILD_VECTORCombine(), performBUILD_VECTORCombine(), performBuildShuffleExtendCombine(), performBuildVectorCombine(), llvm::ARMTargetLowering::PerformCMOVCombine(), performCMovFPCombine(), llvm::ARMTargetLowering::PerformCMOVToBFICombine(), PerformCMPZCombine(), performCONCAT_VECTORSCombine(), performConcatVectorsCombine(), performCONDCombine(), performCSELCombine(), PerformCSETCombine(), llvm::AMDGPUTargetLowering::performCtlz_CttzCombine(), performCTLZCombine(), llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::AMDGPUTargetLowering::PerformDAGCombine(), llvm::ARMTargetLowering::PerformDAGCombine(), llvm::HexagonTargetLowering::PerformDAGCombine(), llvm::LanaiTargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), llvm::PPCTargetLowering::PerformDAGCombine(), llvm::R600TargetLowering::PerformDAGCombine(), llvm::RISCVTargetLowering::PerformDAGCombine(), llvm::SITargetLowering::PerformDAGCombine(), llvm::SparcTargetLowering::PerformDAGCombine(), llvm::SystemZTargetLowering::PerformDAGCombine(), llvm::TargetLowering::PerformDAGCombine(), llvm::VETargetLowering::PerformDAGCombine(), llvm::X86TargetLowering::PerformDAGCombine(), performDivRemCombine(), performDSPShiftCombine(), performDUPCombine(), performDupLane128Combine(), performExtBinopLoadFold(), PerformExtendCombine(), performExtendCombine(), performEXTRACT_VECTOR_ELTCombine(), PerformEXTRACTCombine(), PerformExtractEltCombine(), PerformExtractEltToVMOVRRD(), PerformExtractFpToIntStores(), performExtractLastActiveCombine(), performExtractSubvectorCombine(), performExtractVectorEltCombine(), llvm::AMDGPUTargetLowering::performFAbsCombine(), PerformFADDCombine(), PerformFADDCombine(), performFADDCombine(), PerformFADDCombineWithOperands(), PerformFADDVCMLACombine(), PerformFAddVSelectCombine(), performFirstTrueTestVectorCombine(), performFlagSettingCombine(), PerformFMinMaxCombine(), llvm::AMDGPUTargetLowering::performFNegCombine(), performFP_TO_INT_SATCombine(), performFP_TO_INTCombine(), PerformFPExtendCombine(), performFPExtendCombine(), performFpToIntCombine(), performGatherLoadCombine(), performGLD1Combine(), performGlobalAddressCombine(), PerformHWLoopCombine(), performINSERT_VECTOR_ELTCombine(), PerformInsertEltCombine(), PerformInsertSubvectorCombine(), performInsertSubvectorCombine(), performINTRINSIC_WO_CHAINCombine(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), performIntrinsicCombine(), llvm::AMDGPUTargetLowering::performIntrinsicWOChainCombine(), performIntToFpCombine(), performLastTrueTestVectorCombine(), performLD1Combine(), performLD1ReplicateCombine(), PerformLOADCombine(), llvm::AMDGPUTargetLowering::performLoadCombine(), PerformLongShiftCombine(), performLowerPartialReduction(), performMADD_MSUBCombine(), performMaskedGatherScatterCombine(), performMemPairCombine(), PerformMinMaxCombine(), PerformMinMaxFpToSatCombine(), PerformMinMaxToSatCombine(), performMOVFR2GR_SCombine(), performMOVGR2FR_WCombine(), PerformMULCombine(), PerformMULCombine(), performMULCombine(), performMULCombine(), llvm::AMDGPUTargetLowering::performMulCombine(), performMulCombine(), performMulCombine(), PerformMULCombineWithOperands(), llvm::AMDGPUTargetLowering::performMulhsCombine(), llvm::AMDGPUTargetLowering::performMulhuCombine(), performMULLCombine(), llvm::AMDGPUTargetLowering::performMulLoHiCombine(), performMulVectorCmpZeroCombine(), performMulVectorExtendCombine(), llvm::ARMTargetLowering::PerformMVEExtCombine(), llvm::ARMTargetLowering::PerformMVETruncCombine(), PerformMVEVLDCombine(), PerformMVEVMULLCombine(), performNegCMovCombine(), performNegCSelCombine(), performNEONPostLDSTCombine(), performNVCASTCombine(), PerformORCombine(), performORCombine(), performORCombine(), performORCombine(), performORCombine(), PerformORCombine_i1(), PerformORCombineToBFI(), PerformORCombineToSMULWBT(), performOrXorChainCombine(), performPostLD1Combine(), PerformPREDICATE_CASTCombine(), llvm::AMDGPUTargetLowering::performRcpCombine(), PerformReduceShuffleCombine(), performReinterpretCastCombine(), PerformREMCombine(), performScalarToVectorCombine(), performScatterStoreCombine(), performSELECT_CCCombine(), PerformSELECTCombine(), performSELECTCombine(), performSELECTCombine(), llvm::AMDGPUTargetLowering::performSelectCombine(), performSelectCombine(), performSETCC_BITCASTCombine(), performSetccAddFolding(), PerformSETCCCombine(), performSETCCCombine(), performSETCCCombine(), performSETCCCombine(), performSETCCCombine(), performSETCCCombine(), performSetccMergeZeroCombine(), performSetCCPunpkCombine(), PerformShiftCombine(), PerformSHLCombine(), performSHLCombine(), performSHLCombine(), performSHLCombine(), performSHLCombine(), llvm::AMDGPUTargetLowering::performShlCombine(), PerformSHLSimplify(), PerformShuffleVMOVNCombine(), performSIGN_EXTEND_INREGCombine(), performSignExtendCombine(), PerformSignExtendInregCombine(), performSignExtendInRegCombine(), performSignExtendSetCCCombine(), performSMINCombine(), performSpliceCombine(), performSPLIT_PAIR_F64Combine(), PerformSplittingMVEEXTToWideningLoad(), PerformSplittingMVETruncToNarrowingStores(), PerformSplittingToNarrowingStores(), PerformSplittingToWideningLoad(), performSRACombine(), performSRACombine(), llvm::AMDGPUTargetLowering::performSraCombine(), performSRLCombine(), performSRLCombine(), llvm::AMDGPUTargetLowering::performSrlCombine(), PerformSTORECombine(), llvm::AMDGPUTargetLowering::performStoreCombine(), performSubAddMULCombine(), PerformSUBCombine(), PerformSUBCombine(), performSUBCombine(), PerformSubCSINCCombine(), performSubsToAndsCombine(), performSunpkloCombine(), performSVEAndCombine(), performSVEMulAddSubCombine(), performTBZCombine(), llvm::AMDGPUTargetLowering::performTruncateCombine(), performTruncateCombine(), performTruncateCombine(), PerformTruncatingStoreCombine(), performUADDVAddCombine(), performUADDVCombine(), performUADDVZextCombine(), PerformUMinFpToSatCombine(), PerformUMLALCombine(), performUnpackCombine(), performUzpCombine(), PerformVCMPCombine(), PerformVCVTCombine(), PerformVDUPCombine(), PerformVDUPLANECombine(), PerformVECREDUCE_ADDCombine(), performVecReduceAddCombine(), performVecReduceAddCombineWithUADDLP(), performVecReduceBitwiseCombine(), performVECREDUCECombine(), PerformVECTOR_REG_CASTCombine(), PerformVECTOR_SHUFFLECombine(), performVECTOR_SHUFFLECombine(), performVECTOR_SHUFFLECombine(), performVectorCompareAndMaskUnaryOpCombine(), performVectorDeinterleaveCombine(), performVectorExtCombine(), performVectorExtendCombine(), performVectorExtendToFPCombine(), performVectorNonNegToFPCombine(), performVectorShiftCombine(), performVectorTruncZeroCombine(), performVFMADD_VLCombine(), PerformVLDCombine(), PerformVMOVDRRCombine(), PerformVMOVhrCombine(), PerformVMOVNCombine(), PerformVMOVrhCombine(), PerformVMOVRRDCombine(), performVMSKLTZCombine(), PerformVMULCombine(), PerformVMulVCTPCombine(), performVP_REVERSECombine(), performVP_STORECombine(), performVP_TRUNCATECombine(), PerformVQDMULHCombine(), PerformVQDMULHCombine(), PerformVQMOVNCombine(), PerformVSELECTCombine(), PerformVSELECTCombine(), performVSELECTCombine(), performVSELECTCombine(), performVSelectCombine(), PerformVSetCCToVCTPCombine(), PerformXORCombine(), performXORCombine(), performXorCombine(), performZExtDeinterleaveShuffleCombine(), performZExtUZPCombine(), llvm::SITargetLowering::PostISelFolding(), PrepareTailCall(), llvm::RISCVDAGToDAGISel::PreprocessISelDAG(), promoteExtBeforeAdd(), PromoteMaskArithmetic(), PromoteMaskArithmetic(), promoteToConstantPool(), pushAddIntoCmovOfConsts(), reassociateCSELOperandsForCSE(), llvm::AArch64TargetLowering::ReconstructShuffle(), ReconstructShuffleWithRuntimeMask(), ReconstructTruncateFromBuildVector(), reduceANDOfAtomicLoad(), reduceBuildVecToShuffleWithZero(), reduceMaskedLoadToScalarLoad(), reduceMaskedStoreToScalarStore(), reduceVMULWidth(), reduceVSXSwap(), llvm::SelectionDAG::RemoveDeadNodes(), removeRedundantInsertVectorElt(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::SelectionDAG::ReplaceAllUsesWith(), ReplaceATOMIC_LOAD_128Results(), ReplaceCMP_SWAP_128Results(), ReplaceCMP_SWAP_64Results(), replaceCMP_XCHG_128Results(), replaceInChain(), llvm::AMDGPUTargetLowering::ReplaceNodeResults(), llvm::ARMTargetLowering::ReplaceNodeResults(), llvm::AVRTargetLowering::ReplaceNodeResults(), llvm::LoongArchTargetLowering::ReplaceNodeResults(), llvm::PPCTargetLowering::ReplaceNodeResults(), llvm::R600TargetLowering::ReplaceNodeResults(), llvm::RISCVTargetLowering::ReplaceNodeResults(), llvm::SITargetLowering::ReplaceNodeResults(), llvm::SparcTargetLowering::ReplaceNodeResults(), llvm::X86TargetLowering::ReplaceNodeResults(), replaceSplatVectorStore(), replaceZeroVectorStore(), llvm::SITargetLowering::requiresUniformRegister(), reverseZExtICmpCombine(), llvm::DAGTypeLegalizer::run(), scalarizeBinOpOfSplats(), scalarizeExtEltFP(), scalarizeExtractedBinOp(), llvm::TargetLowering::scalarizeExtractedVectorLoad(), scalarizeVectorStore(), llvm::SDPatternMatch::sd_context_match(), SearchLoopIntrinsic(), llvm::RISCVDAGToDAGISel::Select(), llvm::PPCTargetLowering::SelectAddressRegImm(), llvm::RISCVDAGToDAGISel::SelectAddrRegImm(), llvm::RISCVDAGToDAGISel::SelectAddrRegImmLsb00000(), llvm::RISCVDAGToDAGISel::SelectAddrRegRegScale(), llvm::HexagonDAGToDAGISel::SelectBrevLdIntrinsic(), llvm::AMDGPUDAGToDAGISel::SelectBuildVector(), llvm::SelectionDAGISel::SelectCodeCommon(), selectConstantAddr(), llvm::HexagonDAGToDAGISel::SelectHVXDualOutput(), selectI64Imm(), selectI64ImmDirect(), selectI64ImmDirectPrefix(), selectImm(), selectImmSeq(), llvm::HexagonDAGToDAGISel::SelectIndexedLoad(), llvm::HexagonDAGToDAGISel::SelectIndexedStore(), llvm::HexagonDAGToDAGISel::SelectNewCircIntrinsic(), llvm::PPCTargetLowering::SelectOptimalAddrMode(), SelectSAddrFI(), llvm::RISCVDAGToDAGISel::selectSETCC(), llvm::LoongArchDAGToDAGISel::selectShiftMask(), llvm::RISCVDAGToDAGISel::selectShiftMask(), llvm::RISCVDAGToDAGISel::selectSHXADD_UWOp(), llvm::RISCVDAGToDAGISel::selectSHXADDOp(), llvm::HexagonDAGToDAGISel::SelectVAlign(), llvm::AMDGPUDAGToDAGISel::SelectVectorShuffle(), llvm::RISCVDAGToDAGISel::selectVLSEG(), llvm::RISCVDAGToDAGISel::selectVLSEGFF(), llvm::RISCVDAGToDAGISel::selectVLXSEG(), selectWMMAModsNegAbs(), llvm::FunctionLoweringInfo::set(), ShrinkLoadReplaceStoreWithStore(), simplifyDemandedBitsForPRMT(), llvm::X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(), simplifyDivRem(), llvm::SelectionDAG::simplifyFPBinop(), simplifyMul24(), llvm::TargetLowering::SimplifyMultipleUseDemandedBits(), llvm::TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(), simplifyOp_VL(), llvm::SelectionDAG::simplifySelect(), llvm::TargetLowering::SimplifySetCC(), simplifySetCCWithCTPOP(), llvm::SelectionDAG::simplifyShift(), simplifyShuffleOfShuffle(), sinkProxyReg(), SkipExtensionForVMULL(), llvm::TargetLowering::softenSetCCOperands(), splitAndLowerShuffle(), llvm::VETargetLowering::splitPackedLoadStore(), splitStores(), splitVectorOp(), splitVectorStore(), llvm::HexagonDAGToDAGISel::StoreInstrForLoadIntrinsic(), strictFPExtFromF16(), stripModuloOnShift(), stripTruncAndExt(), takeInexpensiveLog2(), transformAddImmMulImm(), transformAddShlImm(), transformCallee(), truncateAVX512SetCCNoBWI(), truncateVectorWithNARROW(), truncateVectorWithPACK(), tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), tryAdvSIMDModImm321s(), tryAdvSIMDModImm64(), tryAdvSIMDModImm8(), tryAdvSIMDModImmFP(), tryBitfieldInsertOpFromOr(), tryBitfieldInsertOpFromOrAndImm(), tryBuildVectorShuffle(), TryCombineBaseUpdate(), tryCombineCRC32(), tryCombineExtendRShTrunc(), tryCombineFixedPointConvert(), tryCombineLongOpWithDup(), tryCombineMULLWithUZP1(), tryCombineNeonFcvtFP16ToI16(), tryCombineShiftImm(), tryConvertSVEWideCompare(), tryDemorganOfBooleanCondition(), TryDistrubutionADDVecReduce(), tryExtendDUPToExtractHigh(), tryFoldMADwithSRL(), tryFoldSelectIntoOp(), tryFoldToZero(), tryFormConcatFromShuffle(), llvm::HexagonDAGToDAGISel::tryLoadOfLoadIntrinsic(), tryLowerToBSL(), tryLowerToSLI(), TryMatchTrue(), tryMemPairCombine(), TryMULWIDECombine(), llvm::RISCVDAGToDAGISel::tryShrinkShlLogicImm(), trySimplifySrlAddToRshrnb(), trySQDMULHCombine(), trySVESplat64(), trySwapVSelectOperands(), tryToConvertShuffleOfTbl2ToTbl4(), tryToFoldExtendOfConstant(), tryToFoldExtendSelectLoad(), tryToFoldExtOfAtomicLoad(), tryToFoldExtOfExtload(), tryToFoldExtOfLoad(), tryToFoldExtOfMaskedLoad(), tryToReplaceScalarFPConversionWithSVE(), tryToWidenSetCCOperands(), TryWideExtMulCombine(), tryWidenMaskForShuffle(), tryWidenMaskForShuffle(), llvm::X86InstrInfo::unfoldMemoryOperand(), useInversedSetcc(), vectorizeExtractedCast(), visitORCommutative(), Widen(), widenAbs(), widenBuildVec(), widenCtPop(), widenShuffleMask(), widenVectorToPartType(), and llvm::SITargetLowering::wrapAddr64Rsrc().
|
static |
Definition at line 5694 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::APInt::getHighBitsSet(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), isAddSubSExt(), isAddSubZExt(), isSignExtended(), isZeroExtended(), llvm::SelectionDAG::MaskedValueIsZero(), and std::swap().
|
static |
Set the IntrinsicInfo for the aarch64_sve_st<N> intrinsics.
Definition at line 16940 of file AArch64ISelLowering.cpp.
References llvm::CallBase::arg_size(), assert(), DL, llvm::CallBase::getArgOperand(), llvm::Type::getContext(), llvm::TargetLoweringBase::getMemValueType(), llvm::EVT::getScalarType(), llvm::Value::getType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), I, Info, llvm::ISD::INTRINSIC_VOID, and llvm::MachineMemOperand::MOStore.
Referenced by llvm::AArch64TargetLowering::getTgtMemIntrinsic().
|
static |
Definition at line 4092 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::CallingConv::C, LHS, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, and llvm::ISD::SETLT.
Referenced by getAArch64Cmp().
|
static |
Check whether a stack argument requires lowering in a tail call.
Definition at line 9341 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::dyn_cast(), llvm::LoadSDNode::getBasePtr(), llvm::EVT::getFixedSizeInBits(), llvm::MachineFunction::getFrameInfo(), llvm::MemSDNode::getMemoryVT(), llvm::MachineFrameInfo::getObjectOffset(), llvm::MachineFrameInfo::getObjectSize(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::isAssert(), llvm::MachineFrameInfo::isImmutableObjectIndex(), and llvm::ISD::ZERO_EXTEND.
|
static |
Definition at line 4551 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::ConstantSDNode::getZExtValue(), llvm::isPowerOf2_64(), LHS, RHS, llvm::ISD::SETEQ, and llvm::ISD::SETULT.
|
static |
Definition at line 5524 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfSizedIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::EVT::is128BitVector(), N, and llvm::ISD::TRUNCATE.
|
static |
Definition at line 23566 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::cast(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::MemSDNode::getAlign(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::Function::hasMinSize(), llvm::EVT::isFixedLengthVector(), llvm::LSBaseSDNode::isIndexed(), llvm::MemSDNode::isVolatile(), N, replaceSplatVectorStore(), replaceZeroVectorStore(), and SDValue().
|
static |
Definition at line 23245 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::cast(), llvm::commonAlignment(), DL, llvm::MemSDNode::getAlign(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::MachineMemOperand::getFlags(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MachinePointerInfo::getWithOffset(), llvm::isa(), llvm::StoreSDNode::isTruncatingStore(), and llvm::Offset.
Referenced by replaceSplatVectorStore(), and replaceZeroVectorStore().
STATISTIC | ( | NumOptimizedImms | , |
"Number of times immediates were optimized" | ) |
STATISTIC | ( | NumTailCalls | , |
"Number of tail calls" | ) |
|
static |
Definition at line 3252 of file AArch64ISelLowering.cpp.
References assert(), DefMI, MRI, and Reg.
Referenced by llvm::AArch64TargetLowering::fixupPtrauthDiscriminator().
|
static |
Definition at line 14867 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType5(), llvm::AArch64_AM::encodeAdvSIMDModImmType6(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64_AM::isAdvSIMDModImmType5(), llvm::AArch64_AM::isAdvSIMDModImmType6(), llvm::EVT::isFixedLengthVector(), llvm::AArch64Subtarget::isNeonAvailable(), LHS, and SDValue().
Referenced by ConstantBuildVector(), and performANDCombine().
|
static |
Definition at line 14814 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType1(), llvm::AArch64_AM::encodeAdvSIMDModImmType2(), llvm::AArch64_AM::encodeAdvSIMDModImmType3(), llvm::AArch64_AM::encodeAdvSIMDModImmType4(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64_AM::isAdvSIMDModImmType1(), llvm::AArch64_AM::isAdvSIMDModImmType2(), llvm::AArch64_AM::isAdvSIMDModImmType3(), llvm::AArch64_AM::isAdvSIMDModImmType4(), llvm::EVT::isFixedLengthVector(), llvm::AArch64Subtarget::isNeonAvailable(), LHS, and SDValue().
Referenced by ConstantBuildVector(), and performANDCombine().
|
static |
Definition at line 14912 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType7(), llvm::AArch64_AM::encodeAdvSIMDModImmType8(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType7(), llvm::AArch64_AM::isAdvSIMDModImmType8(), and SDValue().
Referenced by ConstantBuildVector().
|
static |
Definition at line 14793 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType10(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType10(), and SDValue().
Referenced by ConstantBuildVector().
|
static |
Definition at line 14943 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType9(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType9(), and SDValue().
Referenced by ConstantBuildVector().
|
static |
Definition at line 14964 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType11(), llvm::AArch64_AM::encodeAdvSIMDModImmType12(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType11(), llvm::AArch64_AM::isAdvSIMDModImmType12(), and SDValue().
Referenced by ConstantBuildVector().
|
static |
Definition at line 22384 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::INTRINSIC_WO_CHAIN, N, and SDValue().
Referenced by performIntrinsicCombine().
|
static |
Definition at line 23710 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPredicateForVector(), llvm::Hi, llvm::Lo, N, and SDValue().
Referenced by performUzpCombine().
|
static |
Definition at line 20950 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, and SDValue().
Referenced by performIntrinsicCombine().
|
static |
Definition at line 22257 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, llvm::Intrinsic::not_intrinsic, RHS, SDValue(), and tryExtendDUPToExtractHigh().
Referenced by performExtendCombine(), performIntrinsicCombine(), and performMULLCombine().
|
static |
Definition at line 27338 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::User::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), llvm::isNullConstant(), llvm::SelectionDAG::isSplatValue(), LHS, N, llvm::SelectionDAG::ReplaceAllUsesWith(), RHS, SDValue(), llvm::SDValue::setNode(), llvm::ISD::TRUNCATE, llvm::SDNode::use_size(), llvm::SDNode::user_begin(), and llvm::SDNode::users().
Referenced by performMULLCombine().
|
static |
Definition at line 22627 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), N, SDValue(), and llvm::ISD::TRUNCATE.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 22290 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::APInt::getSExtValue(), llvm::SelectionDAG::getSignedConstant(), llvm::BuildVectorSDNode::isConstantSplat(), llvm_unreachable, N, llvm::ISD::SCALAR_TO_VECTOR, and SDValue().
Referenced by performIntrinsicCombine().
|
static |
Definition at line 22462 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), getIntrinsicID(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSignedConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm_unreachable, N, SDValue(), llvm::Splat, and llvm::ISD::SPLAT_VECTOR.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 21013 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is64BitVector(), N, and SDValue().
Referenced by performAddSubLongCombine(), and tryCombineLongOpWithDup().
|
static |
Definition at line 14050 of file AArch64ISelLowering.cpp.
References llvm::cast(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), isConcatMask(), and SDValue().
|
static |
Definition at line 15162 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::Add, llvm::ISD::AND, assert(), DL, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SDNode::getAsAPIntVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorNumElements(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::EVT::isScalableVector(), llvm::EVT::isVector(), N, SDValue(), llvm::ISD::SUB, llvm::Sub, and llvm::APInt::trunc().
|
static |
Definition at line 15067 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::CallingConv::C, DL, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), isAllActivePredicate(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), N, SDValue(), X, Y, and llvm::APInt::zextOrTrunc().
|
static |
Definition at line 23750 of file AArch64ISelLowering.cpp.
References canLowerSRLToRoundingShiftForVT(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), llvm::EVT::isScalableVector(), and SDValue().
Referenced by if(), and performUzpCombine().
|
static |
Definition at line 21629 of file AArch64ISelLowering.cpp.
References DL, llvm::EVT::getFixedSizeInBits(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::ConstantSDNode::getSExtValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::isConstOrConstSplat(), llvm::EVT::isPow2VectorType(), llvm::EVT::isScalableVector(), llvm::EVT::isVector(), llvm::ISD::MUL, Mul, N, SDValue(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SMIN, and llvm::ISD::SRA.
Referenced by performSMINCombine().
|
static |
Definition at line 15337 of file AArch64ISelLowering.cpp.
References convertFromScalableVector(), DL, llvm::SelectionDAG::getConstant(), llvm::APInt::getHiBits(), llvm::APInt::getLoBits(), llvm::SelectionDAG::getNode(), llvm::APInt::getSExtValue(), llvm::SelectionDAG::getSplatVector(), llvm::EVT::is128BitVector(), llvm::AArch64_AM::isSVECpyDupImm(), SDValue(), and llvm::APInt::trunc().
Referenced by ConstantBuildVector().
|
static |
Definition at line 26385 of file AArch64ISelLowering.cpp.
References llvm::cast(), llvm::ISD::FADD, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), N, SDValue(), llvm::ISD::SETCC, and llvm::ISD::VSELECT.
Referenced by performVSelectCombine().
|
static |
Definition at line 14433 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, DL, llvm::dyn_cast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), I, llvm::ISD::INTRINSIC_WO_CHAIN, and SDValue().
|
static |
Tries to replace scalar FP <-> INT conversions with SVE in streaming functions, this can help to reduce the number of fmovs to/from GPRs.
Definition at line 19806 of file AArch64ISelLowering.cpp.
References llvm::EVT::bitsGT(), llvm::EVT::changeVectorElementType(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::ISD::INSERT_VECTOR_ELT, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::AArch64Subtarget::isStreaming(), llvm::AArch64Subtarget::isStreamingCompatible(), isSupportedType(), llvm::AArch64Subtarget::isSVEorStreamingSVEAvailable(), llvm::EVT::isVector(), N, and SDValue().
Referenced by performFpToIntCombine(), and performIntToFpCombine().
|
static |
Definition at line 25996 of file AArch64ISelLowering.cpp.
References llvm::any_of(), llvm::cast(), llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDNode::getOpcode(), llvm::EVT::getScalarSizeInBits(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), N, SDValue(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::ISD::VSELECT, and llvm::ISD::ZERO_EXTEND.
Referenced by performSETCCCombine().
|
static |
Definition at line 14400 of file AArch64ISelLowering.cpp.
References llvm::cast(), DL, llvm::SelectionDAG::getBitcast(), llvm::EVT::getFixedSizeInBits(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), SDValue(), and llvm::widenShuffleMaskElts().
Referenced by lowerVECTOR_SHUFFLE().
|
static |
Definition at line 29254 of file AArch64ISelLowering.cpp.
References llvm::IRBuilderBase::CreateCall(), llvm::IRBuilderBase::CreateConstGEP1_32(), llvm::IRBuilderBase::CreatePointerCast(), llvm::IRBuilderBase::GetInsertBlock(), llvm::IRBuilderBase::getInt8Ty(), llvm::Intrinsic::getOrInsertDeclaration(), llvm::BasicBlock::getParent(), llvm::GlobalValue::getParent(), llvm::IRBuilderBase::getPtrTy(), and llvm::Offset.
Referenced by llvm::AArch64TargetLowering::getIRStackGuard(), and llvm::AArch64TargetLowering::getSafeStackPointerLocation().
|
static |
Definition at line 4431 of file AArch64ISelLowering.cpp.
References DL, FlagsVT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and llvm::SelectionDAG::getVTList().
Referenced by lowerADDSUBO_CARRY().
|
static |
WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class.
Definition at line 13329 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), and llvm::ISD::INSERT_SUBVECTOR.
Referenced by constructDup(), GeneratePerfectShuffle(), and performConcatVectorsCombine().
|
constexpr |
Value type used for condition codes.
Definition at line 166 of file AArch64ISelLowering.cpp.
Referenced by getCondCode().
cl::opt< bool > EnableAArch64ELFLocalDynamicTLSGeneration("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false)) | ( | "aarch64-elf-ldtls-generation" | , |
cl::Hidden | , | ||
cl::desc("Allow AArch64 Local Dynamic TLS code generation") | , | ||
cl::init(false) | ) |
Referenced by llvm::AArch64MCInstLower::lowerSymbolOperandELF().
|
static |
Referenced by performSignExtendInRegCombine(), and performSVEAndCombine().
|
static |
Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant().
|
constexpr |
Value type used for NZCV flags.
Definition at line 169 of file AArch64ISelLowering.cpp.
Referenced by emitComparison(), emitConditionalComparison(), emitStrictFPComparison(), Expand64BitShift(), getAArch64XALUOOp(), lowerADDSUBO_CARRY(), performANDORCSELCombine(), performRNDRCombine(), reassociateCSELOperandsForCSE(), valueToCarryFlag(), and llvm::AArch64SelectionDAGInfo::verifyTargetNode().
Definition at line 174 of file AArch64ISelLowering.cpp.
Referenced by llvm::AArch64::getFPRArgRegs().
Definition at line 171 of file AArch64ISelLowering.cpp.
Referenced by f64AssignAAPCS(), and llvm::AArch64::getGPRArgRegs().
|
static |
Referenced by isOrXorChain().
true |
Simplify Addr given that the top byte of it is ignored by HW during / address translation.
static bool performTBISimplification(SDValue Addr, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { APInt DemandedMask = APInt::getLowBitsSet(64, 56); KnownBits Known; TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), !DCI.isBeforeLegalizeOps()); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (TLI.SimplifyDemandedBits(Addr, DemandedMask, Known, TLO)) { DCI.CommitTargetLoweringOpt(TLO); return true; } return false; }
static SDValue foldTruncStoreOfExt(SelectionDAG &DAG, SDNode *N) { assert((N->getOpcode() == ISD::STORE || N->getOpcode() == ISD::MSTORE) && "Expected STORE dag node in input!");
if (auto Store = dyn_cast<StoreSDNode>(N)) { if (!Store->isTruncatingStore() || Store->isIndexed()) return SDValue(); SDValue Ext = Store->getValue(); auto ExtOpCode = Ext.getOpcode(); if (ExtOpCode != ISD::ZERO_EXTEND && ExtOpCode != ISD::SIGN_EXTEND && ExtOpCode != ISD::ANY_EXTEND) return SDValue(); SDValue Orig = Ext->getOperand(0); if (Store->getMemoryVT() != Orig.getValueType()) return SDValue(); return DAG.getStore(Store->getChain(), SDLoc(Store), Orig, Store->getBasePtr(), Store->getMemOperand()); }
return SDValue(); }
A custom combine to lower load <3 x i8> as the more efficient sequence below: ldrb wX, [x0, #2] ldrh wY, [x0] orr wX, wY, wX, lsl #16 fmov s0, wX
Note that an alternative sequence with even fewer (although usually more complex/expensive) instructions would be: ld1r.4h { v0 }, [x0], #2 ld1.b { v0 }[2], [x0]
Generating this sequence unfortunately results in noticeably worse codegen for code that extends the loaded v3i8, due to legalization breaking vector shuffle detection in a way that is very difficult to work around. TODO: Revisit once v3i8 legalization has been improved in general. static SDValue combineV3I8LoadExt(LoadSDNode *LD, SelectionDAG &DAG) { EVT MemVT = LD->getMemoryVT(); if (MemVT != EVT::getVectorVT(*DAG.getContext(), MVT::i8, 3) || LD->getBaseAlign() >= 4) return SDValue();
SDLoc DL(LD); MachineFunction &MF = DAG.getMachineFunction(); SDValue Chain = LD->getChain(); SDValue BasePtr = LD->getBasePtr(); MachineMemOperand *MMO = LD->getMemOperand(); assert(LD->getOffset().isUndef() && "undef offset expected");
Load 2 x i8, then 1 x i8. SDValue L16 = DAG.getLoad(MVT::i16, DL, Chain, BasePtr, MMO); TypeSize Offset2 = TypeSize::getFixed(2); SDValue L8 = DAG.getLoad(MVT::i8, DL, Chain, DAG.getMemBasePlusOffset(BasePtr, Offset2, DL), MF.getMachineMemOperand(MMO, 2, 1));
Extend to i32. SDValue Ext16 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, L16); SDValue Ext8 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, L8);
Pack 2 x i8 and 1 x i8 in an i32 and convert to v4i8. SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i32, Ext8, DAG.getConstant(16, DL, MVT::i32)); SDValue Or = DAG.getNode(ISD::OR, DL, MVT::i32, Ext16, Shl); SDValue Cast = DAG.getNode(ISD::BITCAST, DL, MVT::v4i8, Or);
Extract v3i8 again. SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MemVT, Cast, DAG.getConstant(0, DL, MVT::i64)); SDValue TokenFactor = DAG.getNode( ISD::TokenFactor, DL, MVT::Other, {SDValue(cast<SDNode>(L16), 1), SDValue(cast<SDNode>(L8), 1)}); return DAG.getMergeValues({Extract, TokenFactor}, DL); }
Perform TBI simplification if supported by the target and try to break up nontemporal loads larger than 256-bits loads for odd types so LDNPQ 256-bit load instructions can be selected. static SDValue performLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { if (Subtarget->supportsAddressTopByteIgnored()) performTBISimplification(N->getOperand(1), DCI, DAG);
LoadSDNode *LD = cast<LoadSDNode>(N); EVT RegVT = LD->getValueType(0); EVT MemVT = LD->getMemoryVT(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDLoc DL(LD);
Cast ptr32 and ptr64 pointers to the default address space before a load. unsigned AddrSpace = LD->getAddressSpace(); if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR || AddrSpace == ARM64AS::PTR32_UPTR) { MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); if (PtrVT != LD->getBasePtr().getSimpleValueType()) { SDValue Cast = DAG.getAddrSpaceCast(DL, PtrVT, LD->getBasePtr(), AddrSpace, 0); return DAG.getExtLoad(LD->getExtensionType(), DL, RegVT, LD->getChain(), Cast, LD->getPointerInfo(), MemVT, LD->getBaseAlign(), LD->getMemOperand()->getFlags()); } }
if (LD->isVolatile() || !Subtarget->isLittleEndian()) return SDValue(N, 0);
if (SDValue Res = combineV3I8LoadExt(LD, DAG)) return Res;
if (!LD->isNonTemporal()) return SDValue(N, 0);
if (MemVT.isScalableVector() || MemVT.getSizeInBits() <= 256 || MemVT.getSizeInBits() % 256 == 0 || 256 % MemVT.getScalarSizeInBits() != 0) return SDValue(N, 0);
SDValue Chain = LD->getChain(); SDValue BasePtr = LD->getBasePtr(); SDNodeFlags Flags = LD->getFlags(); SmallVector<SDValue, 4> LoadOps; SmallVector<SDValue, 4> LoadOpsChain; Replace any non temporal load over 256-bit with a series of 256 bit loads and a scalar/vector load less than 256. This way we can utilize 256-bit loads and reduce the amount of load instructions generated. MVT NewVT = MVT::getVectorVT(MemVT.getVectorElementType().getSimpleVT(), 256 / MemVT.getVectorElementType().getSizeInBits()); unsigned Num256Loads = MemVT.getSizeInBits() / 256; Create all 256-bit loads starting from offset 0 and up to Num256Loads-1*32. for (unsigned I = 0; I < Num256Loads; I++) { unsigned PtrOffset = I * 32; SDValue NewPtr = DAG.getMemBasePlusOffset( BasePtr, TypeSize::getFixed(PtrOffset), DL, Flags); Align NewAlign = commonAlignment(LD->getAlign(), PtrOffset); SDValue NewLoad = DAG.getLoad( NewVT, DL, Chain, NewPtr, LD->getPointerInfo().getWithOffset(PtrOffset), NewAlign, LD->getMemOperand()->getFlags(), LD->getAAInfo()); LoadOps.push_back(NewLoad); LoadOpsChain.push_back(SDValue(cast<SDNode>(NewLoad), 1)); }
Process remaining bits of the load operation. This is done by creating an UNDEF vector to match the size of the 256-bit loads and inserting the remaining load to it. We extract the original load type at the end using EXTRACT_SUBVECTOR instruction. unsigned BitsRemaining = MemVT.getSizeInBits() % 256; unsigned PtrOffset = (MemVT.getSizeInBits() - BitsRemaining) / 8; MVT RemainingVT = MVT::getVectorVT( MemVT.getVectorElementType().getSimpleVT(), BitsRemaining / MemVT.getVectorElementType().getSizeInBits()); SDValue NewPtr = DAG.getMemBasePlusOffset( BasePtr, TypeSize::getFixed(PtrOffset), DL, Flags); Align NewAlign = commonAlignment(LD->getAlign(), PtrOffset); SDValue RemainingLoad = DAG.getLoad(RemainingVT, DL, Chain, NewPtr, LD->getPointerInfo().getWithOffset(PtrOffset), NewAlign, LD->getMemOperand()->getFlags(), LD->getAAInfo()); SDValue UndefVector = DAG.getUNDEF(NewVT); SDValue InsertIdx = DAG.getVectorIdxConstant(0, DL); SDValue ExtendedRemainingLoad = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, NewVT, {UndefVector, RemainingLoad, InsertIdx}); LoadOps.push_back(ExtendedRemainingLoad); LoadOpsChain.push_back(SDValue(cast<SDNode>(RemainingLoad), 1)); EVT ConcatVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), LoadOps.size() * NewVT.getVectorNumElements()); SDValue ConcatVectors = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, LoadOps); Extract the original vector type size. SDValue ExtractSubVector = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MemVT, {ConcatVectors, DAG.getVectorIdxConstant(0, DL)}); SDValue TokenFactor = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, LoadOpsChain); return DAG.getMergeValues({ExtractSubVector, TokenFactor}, DL); }
static EVT tryGetOriginalBoolVectorType(SDValue Op, int Depth = 0) { EVT VecVT = Op.getValueType(); assert(VecVT.isVector() && VecVT.getVectorElementType() == MVT::i1 && "Need boolean vector type.");
if (Depth > 3) return MVT::INVALID_SIMPLE_VALUE_TYPE;
We can get the base type from a vector compare or truncate. if (Op.getOpcode() == ISD::SETCC || Op.getOpcode() == ISD::TRUNCATE) return Op.getOperand(0).getValueType();
If an operand is a bool vector, continue looking. EVT BaseVT = MVT::INVALID_SIMPLE_VALUE_TYPE; for (SDValue Operand : Op->op_values()) { if (Operand.getValueType() != VecVT) continue;
EVT OperandVT = tryGetOriginalBoolVectorType(Operand, Depth + 1); if (!BaseVT.isSimple()) BaseVT = OperandVT; else if (OperandVT != BaseVT) return MVT::INVALID_SIMPLE_VALUE_TYPE; }
return BaseVT; }
When converting a <N x iX> vector to <N x i1> to store or use as a scalar iN, we can use a trick that extracts the i^th bit from the i^th element and then performs a vector add to get a scalar bitmask. This requires that each element's bits are either all 1 or all 0. static SDValue vectorToScalarBitmask(SDNode *N, SelectionDAG &DAG) { SDLoc DL(N); SDValue ComparisonResult(N, 0); EVT VecVT = ComparisonResult.getValueType(); assert(VecVT.isVector() && "Must be a vector type");
unsigned NumElts = VecVT.getVectorNumElements(); if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) return SDValue();
if (VecVT.getVectorElementType() != MVT::i1 && !DAG.getTargetLoweringInfo().isTypeLegal(VecVT)) return SDValue();
If we can find the original types to work on instead of a vector of i1, we can avoid extend/extract conversion instructions. if (VecVT.getVectorElementType() == MVT::i1) { VecVT = tryGetOriginalBoolVectorType(ComparisonResult); if (!VecVT.isSimple()) { unsigned BitsPerElement = std::max(64 / NumElts, 8u); // >= 64-bit vector VecVT = MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement), NumElts); } } VecVT = VecVT.changeVectorElementTypeToInteger();
Large vectors don't map directly to this conversion, so to avoid too many edge cases, we don't apply it here. The conversion will likely still be applied later via multiple smaller vectors, whose results are concatenated. if (VecVT.getSizeInBits() > 128) return SDValue();
Ensure that all elements' bits are either 0s or 1s. ComparisonResult = DAG.getSExtOrTrunc(ComparisonResult, DL, VecVT);
bool IsLE = DAG.getDataLayout().isLittleEndian(); SmallVector<SDValue, 16> MaskConstants; if (DAG.getSubtarget<AArch64Subtarget>().isNeonAvailable() && VecVT == MVT::v16i8) { v16i8 is a special case, as we have 16 entries but only 8 positional bits per entry. We split it into two halves, apply the mask, zip the halves to create 8x 16-bit values, and the perform the vector reduce. for (unsigned Half = 0; Half < 2; ++Half) { for (unsigned I = 0; I < 8; ++I) { On big-endian targets, the lane order in sub-byte vector elements gets reversed, so we need to flip the bit index. unsigned MaskBit = IsLE ? (1u << I) : (1u << (7 - I)); MaskConstants.push_back(DAG.getConstant(MaskBit, DL, MVT::i32)); } } SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, MaskConstants); SDValue RepresentativeBits = DAG.getNode(ISD::AND, DL, VecVT, ComparisonResult, Mask);
SDValue UpperRepresentativeBits = DAG.getNode(AArch64ISD::EXT, DL, VecVT, RepresentativeBits, RepresentativeBits, DAG.getConstant(8, DL, MVT::i32)); SDValue Zipped = DAG.getNode(AArch64ISD::ZIP1, DL, VecVT, RepresentativeBits, UpperRepresentativeBits); Zipped = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, Zipped); return DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i16, Zipped); }
All other vector sizes. unsigned NumEl = VecVT.getVectorNumElements(); for (unsigned I = 0; I < NumEl; ++I) { unsigned MaskBit = IsLE ? (1u << I) : (1u << (NumEl - 1 - I)); MaskConstants.push_back(DAG.getConstant(MaskBit, DL, MVT::i64)); }
SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, MaskConstants); SDValue RepresentativeBits = DAG.getNode(ISD::AND, DL, VecVT, ComparisonResult, Mask); EVT ResultVT = MVT::getIntegerVT(std::max<unsigned>( NumElts, VecVT.getVectorElementType().getSizeInBits())); return DAG.getNode(ISD::VECREDUCE_ADD, DL, ResultVT, RepresentativeBits); }
static SDValue combineBoolVectorAndTruncateStore(SelectionDAG &DAG, StoreSDNode *Store) { if (!Store->isTruncatingStore()) return SDValue();
SDLoc DL(Store); SDValue VecOp = Store->getValue(); EVT VT = VecOp.getValueType(); EVT MemVT = Store->getMemoryVT();
if (!MemVT.isVector() || !VT.isVector() || MemVT.getVectorElementType() != MVT::i1) return SDValue();
If we are storing a vector that we are currently building, let scalarizeVectorStore() handle this more efficiently. if (VecOp.getOpcode() == ISD::BUILD_VECTOR) return SDValue();
VecOp = DAG.getNode(ISD::TRUNCATE, DL, MemVT, VecOp); SDValue VectorBits = vectorToScalarBitmask(VecOp.getNode(), DAG); if (!VectorBits) return SDValue();
EVT StoreVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getStoreSizeInBits()); SDValue ExtendedBits = DAG.getZExtOrTrunc(VectorBits, DL, StoreVT); return DAG.getStore(Store->getChain(), DL, ExtendedBits, Store->getBasePtr(), Store->getMemOperand()); }
Combine store (fp_to_int X) to use vector semantics around the conversion when NEON is available. This allows us to store the in-vector result directly without transferring the result into a GPR in the process. static SDValue combineStoreValueFPToInt(StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { Limit to post-legalization in order to avoid peeling truncating stores. if (DCI.isBeforeLegalize()) return SDValue(); if (!Subtarget->isNeonAvailable()) return SDValue(); Source operand is already a vector. SDValue Value = ST->getValue(); if (Value.getValueType().isVector()) return SDValue();
Look through potential assertions. while (Value->isAssert()) Value = Value.getOperand(0);
if (Value.getOpcode() != ISD::FP_TO_SINT && Value.getOpcode() != ISD::FP_TO_UINT) return SDValue(); if (!Value->hasOneUse()) return SDValue();
SDValue FPSrc = Value.getOperand(0); EVT SrcVT = FPSrc.getValueType(); if (SrcVT != MVT::f32 && SrcVT != MVT::f64) return SDValue();
No support for assignments such as i64 = fp_to_sint i32 EVT VT = Value.getSimpleValueType(); if (VT != SrcVT.changeTypeToInteger()) return SDValue();
Create a 128-bit element vector to avoid widening. The floating point conversion is transformed into a single element conversion via a pattern. unsigned NumElements = 128 / SrcVT.getFixedSizeInBits(); EVT VecSrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT, NumElements); EVT VecDstVT = VecSrcVT.changeTypeToInteger(); SDLoc DL(ST); SDValue VecFP = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, FPSrc); SDValue VecConv = DAG.getNode(Value.getOpcode(), DL, VecDstVT, VecFP);
SDValue Zero = DAG.getVectorIdxConstant(0, DL); SDValue Extracted = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, VecConv, Zero);
DCI.CombineTo(ST->getValue().getNode(), Extracted); return SDValue(ST, 0); }
bool isHalvingTruncateOfLegalScalableType(EVT SrcVT, EVT DstVT) { return (SrcVT == MVT::nxv8i16 && DstVT == MVT::nxv8i8) || (SrcVT == MVT::nxv4i32 && DstVT == MVT::nxv4i16) || (SrcVT == MVT::nxv2i64 && DstVT == MVT::nxv2i32); }
Combine store (trunc X to <3 x i8>) to sequence of ST1.b. static SDValue combineI8TruncStore(StoreSDNode *ST, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { SDValue Value = ST->getValue(); EVT ValueVT = Value.getValueType();
if (ST->isVolatile() || !Subtarget->isLittleEndian() || Value.getOpcode() != ISD::TRUNCATE || ValueVT != EVT::getVectorVT(*DAG.getContext(), MVT::i8, 3)) return SDValue();
assert(ST->getOffset().isUndef() && "undef offset expected"); SDLoc DL(ST); auto WideVT = EVT::getVectorVT( DAG.getContext(), Value->getOperand(0).getValueType().getVectorElementType(), 4); SDValue UndefVector = DAG.getUNDEF(WideVT); SDValue WideTrunc = DAG.getNode( ISD::INSERT_SUBVECTOR, DL, WideVT, {UndefVector, Value->getOperand(0), DAG.getVectorIdxConstant(0, DL)}); SDValue Cast = DAG.getNode( ISD::BITCAST, DL, WideVT.getSizeInBits() == 64 ? MVT::v8i8 : MVT::v16i8, WideTrunc);
MachineFunction &MF = DAG.getMachineFunction(); SDValue Chain = ST->getChain(); MachineMemOperand *MMO = ST->getMemOperand(); unsigned IdxScale = WideVT.getScalarSizeInBits() / 8; SDValue E2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, Cast, DAG.getConstant(2 * IdxScale, DL, MVT::i64)); TypeSize Offset2 = TypeSize::getFixed(2); SDValue Ptr2 = DAG.getMemBasePlusOffset(ST->getBasePtr(), Offset2, DL); Chain = DAG.getStore(Chain, DL, E2, Ptr2, MF.getMachineMemOperand(MMO, 2, 1));
SDValue E1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, Cast, DAG.getConstant(1 * IdxScale, DL, MVT::i64)); TypeSize Offset1 = TypeSize::getFixed(1); SDValue Ptr1 = DAG.getMemBasePlusOffset(ST->getBasePtr(), Offset1, DL); Chain = DAG.getStore(Chain, DL, E1, Ptr1, MF.getMachineMemOperand(MMO, 1, 1));
SDValue E0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i8, Cast, DAG.getConstant(0, DL, MVT::i64)); Chain = DAG.getStore(Chain, DL, E0, ST->getBasePtr(), MF.getMachineMemOperand(MMO, 0, 1)); return Chain; }
static unsigned getFPSubregForVT(EVT VT) { assert(VT.isSimple() && "Expected simple VT"); switch (VT.getSimpleVT().SimpleTy) { case MVT::aarch64mfp8: return AArch64::bsub; case MVT::f16: return AArch64::hsub; case MVT::f32: return AArch64::ssub; case MVT::f64: return AArch64::dsub; default: llvm_unreachable("Unexpected VT!"); } }
static SDValue performSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { StoreSDNode *ST = cast<StoreSDNode>(N); SDValue Chain = ST->getChain(); SDValue Value = ST->getValue(); SDValue Ptr = ST->getBasePtr(); EVT ValueVT = Value.getValueType(); EVT MemVT = ST->getMemoryVT(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDLoc DL(ST);
if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget)) return Res;
auto hasValidElementTypeForFPTruncStore = [](EVT VT) { EVT EltVT = VT.getVectorElementType(); return EltVT == MVT::f32 || EltVT == MVT::f64; };
Cast ptr32 and ptr64 pointers to the default address space before a store. unsigned AddrSpace = ST->getAddressSpace(); if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR || AddrSpace == ARM64AS::PTR32_UPTR) { MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); if (PtrVT != Ptr.getSimpleValueType()) { SDValue Cast = DAG.getAddrSpaceCast(DL, PtrVT, Ptr, AddrSpace, 0); return DAG.getStore(Chain, DL, Value, Cast, ST->getPointerInfo(), ST->getBaseAlign(), ST->getMemOperand()->getFlags(), ST->getAAInfo()); } }
if (SDValue Res = combineI8TruncStore(ST, DAG, Subtarget)) return Res;
If this is an FP_ROUND followed by a store, fold this into a truncating store. We can do this even if this is already a truncstore. We purposefully don't care about legality of the nodes here as we know they can be split down into something legal. if (DCI.isBeforeLegalizeOps() && Value.getOpcode() == ISD::FP_ROUND && Value.getNode()->hasOneUse() && ST->isUnindexed() && Subtarget->useSVEForFixedLengthVectors() && ValueVT.isFixedLengthVector() && ValueVT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits() && hasValidElementTypeForFPTruncStore(Value.getOperand(0).getValueType())) return DAG.getTruncStore(Chain, DL, Value.getOperand(0), Ptr, MemVT, ST->getMemOperand());
if (SDValue Split = splitStores(N, DCI, DAG, Subtarget)) return Split;
if (Subtarget->supportsAddressTopByteIgnored() && performTBISimplification(N->getOperand(2), DCI, DAG)) return SDValue(N, 0);
if (SDValue Store = foldTruncStoreOfExt(DAG, N)) return Store;
if (SDValue Store = combineBoolVectorAndTruncateStore(DAG, ST)) return Store;
if (ST->isTruncatingStore() && isHalvingTruncateOfLegalScalableType(ValueVT, MemVT)) { if (SDValue Rshrnb = trySimplifySrlAddToRshrnb(ST->getOperand(1), DAG, Subtarget)) { return DAG.getTruncStore(ST->getChain(), ST, Rshrnb, ST->getBasePtr(), MemVT, ST->getMemOperand()); } }
This is an integer vector_extract_elt followed by a (possibly truncating) store. We may be able to replace this with a store of an FP subregister. if (DCI.isAfterLegalizeDAG() && ST->isUnindexed() && Value.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
SDValue Vector = Value.getOperand(0); SDValue ExtIdx = Value.getOperand(1); EVT VectorVT = Vector.getValueType(); EVT ElemVT = VectorVT.getVectorElementType();
if (!ValueVT.isInteger()) return SDValue();
Propagate zero constants (applying this fold may miss optimizations). if (ISD::isConstantSplatVectorAllZeros(Vector.getNode())) { SDValue ZeroElt = DAG.getConstant(0, DL, ValueVT); DAG.ReplaceAllUsesWith(Value, ZeroElt); return SDValue(); }
if (ValueVT != MemVT && !ST->isTruncatingStore()) return SDValue();
This could generate an additional extract if the index is non-zero and the extracted value has multiple uses. auto *ExtCst = dyn_cast<ConstantSDNode>(ExtIdx); if ((!ExtCst || !ExtCst->isZero()) && !Value.hasOneUse()) return SDValue();
These can lower to st1, which is preferable if we're unlikely to fold the addressing into the store. if (Subtarget->isNeonAvailable() && ElemVT == MemVT && (VectorVT.is64BitVector() || VectorVT.is128BitVector()) && ExtCst && !ExtCst->isZero() && ST->getBasePtr().getOpcode() != ISD::ADD) return SDValue();
if (MemVT == MVT::i64 || MemVT == MVT::i32) { Heuristic: If there are other users of w/x integer scalars extracted from this vector that won't fold into the store – abandon folding. Applying this fold may disrupt paired stores. for (const auto &Use : Vector->uses()) { if (Use.getResNo() != Vector.getResNo()) continue; const SDNode *User = Use.getUser(); if (User->getOpcode() == ISD::EXTRACT_VECTOR_ELT && (!User->hasOneUse() || (*User->user_begin())->getOpcode() != ISD::STORE)) return SDValue(); } }
SDValue ExtVector = Vector; if (!ExtCst || !ExtCst->isZero()) { Handle extracting from lanes != 0. SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Value.getValueType(), Vector, ExtIdx); SDValue Zero = DAG.getVectorIdxConstant(0, DL); ExtVector = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VectorVT, DAG.getUNDEF(VectorVT), Ext, Zero); }
EVT FPMemVT = MemVT == MVT::i8 ? MVT::aarch64mfp8 : EVT::getFloatingPointVT(MemVT.getSizeInBits()); SDValue FPSubreg = DAG.getTargetExtractSubreg(getFPSubregForVT(FPMemVT), DL, FPMemVT, ExtVector);
return DAG.getStore(ST->getChain(), DL, FPSubreg, ST->getBasePtr(), ST->getMemOperand()); }
return SDValue(); }
static bool isSequentialConcatOfVectorInterleave(SDNode *N, SmallVectorImpl<SDValue> &Ops) { if (N->getOpcode() != ISD::CONCAT_VECTORS) return false;
unsigned NumParts = N->getNumOperands();
We should be concatenating each sequential result from a VECTOR_INTERLEAVE. SDNode *InterleaveOp = N->getOperand(0).getNode(); if (InterleaveOp->getOpcode() != ISD::VECTOR_INTERLEAVE || InterleaveOp->getNumOperands() != NumParts) return false;
for (unsigned I = 0; I < NumParts; I++) if (N->getOperand(I) != SDValue(InterleaveOp, I)) return false;
Ops.append(InterleaveOp->op_begin(), InterleaveOp->op_end()); return true; }
static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL, SDValue WideMask, unsigned RequiredNumParts) { if (WideMask->getOpcode() == ISD::CONCAT_VECTORS) { SmallVector<SDValue, 4> MaskInterleaveOps; if (!isSequentialConcatOfVectorInterleave(WideMask.getNode(), MaskInterleaveOps)) return SDValue();
if (MaskInterleaveOps.size() != RequiredNumParts) return SDValue();
Make sure the inputs to the vector interleave are identical. if (!llvm::all_equal(MaskInterleaveOps)) return SDValue();
return MaskInterleaveOps[0]; }
if (WideMask->getOpcode() != ISD::SPLAT_VECTOR) return SDValue();
ElementCount EC = WideMask.getValueType().getVectorElementCount(); assert(EC.isKnownMultipleOf(RequiredNumParts) && "Expected element count divisible by number of parts"); EC = EC.divideCoefficientBy(RequiredNumParts); return DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::getVectorVT(MVT::i1, EC), WideMask->getOperand(0)); }
static SDValue performInterleavedMaskedStoreCombine( SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { if (!DCI.isBeforeLegalize()) return SDValue();
MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); SDValue WideValue = MST->getValue();
Bail out if the stored value has an unexpected number of uses, since we'll have to perform manual interleaving and may as well just use normal masked stores. Also, discard masked stores that are truncating or indexed. if (!WideValue.hasOneUse() || !ISD::isNormalMaskedStore(MST) || !MST->isSimple() || !MST->getOffset().isUndef()) return SDValue();
SmallVector<SDValue, 4> ValueInterleaveOps; if (!isSequentialConcatOfVectorInterleave(WideValue.getNode(), ValueInterleaveOps)) return SDValue();
unsigned NumParts = ValueInterleaveOps.size(); if (NumParts != 2 && NumParts != 4) return SDValue();
At the moment we're unlikely to see a fixed-width vector interleave as we usually generate shuffles instead. EVT SubVecTy = ValueInterleaveOps[0].getValueType(); if (!SubVecTy.isScalableVT() || SubVecTy.getSizeInBits().getKnownMinValue() != 128 || !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy)) return SDValue();
SDLoc DL(N); SDValue NarrowMask = getNarrowMaskForInterleavedOps(DAG, DL, MST->getMask(), NumParts); if (!NarrowMask) return SDValue();
const Intrinsic::ID IID = NumParts == 2 ? Intrinsic::aarch64_sve_st2 : Intrinsic::aarch64_sve_st4; SmallVector<SDValue, 8> NewStOps; NewStOps.append({MST->getChain(), DAG.getConstant(IID, DL, MVT::i32)}); NewStOps.append(ValueInterleaveOps); NewStOps.append({NarrowMask, MST->getBasePtr()}); return DAG.getNode(ISD::INTRINSIC_VOID, DL, MVT::Other, NewStOps); }
static SDValue performMSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { MaskedStoreSDNode *MST = cast<MaskedStoreSDNode>(N); SDValue Value = MST->getValue(); SDValue Mask = MST->getMask(); SDLoc DL(N);
if (SDValue Res = performInterleavedMaskedStoreCombine(N, DCI, DAG)) return Res;
If this is a UZP1 followed by a masked store, fold this into a masked truncating store. We can do this even if this is already a masked truncstore. if (Value.getOpcode() == AArch64ISD::UZP1 && Value->hasOneUse() && MST->isUnindexed() && Mask->getOpcode() == AArch64ISD::PTRUE && Value.getValueType().isInteger()) { Value = Value.getOperand(0); if (Value.getOpcode() == ISD::BITCAST) { EVT HalfVT = Value.getValueType().getHalfNumVectorElementsVT(*DAG.getContext()); EVT InVT = Value.getOperand(0).getValueType();
if (HalfVT.widenIntegerVectorElementType(*DAG.getContext()) == InVT) { unsigned MinSVESize = Subtarget->getMinSVEVectorSizeInBits(); unsigned PgPattern = Mask->getConstantOperandVal(0);
Ensure we can double the size of the predicate pattern unsigned NumElts = getNumElementsFromSVEPredPattern(PgPattern); if (NumElts && NumElts * InVT.getVectorElementType().getSizeInBits() <= MinSVESize) { Mask = getPTrue(DAG, DL, InVT.changeVectorElementType(MVT::i1), PgPattern); return DAG.getMaskedStore(MST->getChain(), DL, Value.getOperand(0), MST->getBasePtr(), MST->getOffset(), Mask, MST->getMemoryVT(), MST->getMemOperand(), MST->getAddressingMode(), /*IsTruncating=
Definition at line 24941 of file AArch64ISelLowering.cpp.
|
static |
Referenced by llvm::AArch64TargetLowering::shouldPreservePtrArith().