LLVM 19.0.0git
|
#include "AArch64ISelLowering.h"
#include "AArch64CallingConvention.h"
#include "AArch64ExpandImm.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64PerfectShuffle.h"
#include "AArch64RegisterInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/ObjCARCUtil.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RuntimeLibcalls.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/TargetCallingConv.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GetElementPtrTypeIterator.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsAArch64.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
#include "llvm/MC/MCRegisterInfo.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include "llvm/TargetParser/Triple.h"
#include <algorithm>
#include <bitset>
#include <cassert>
#include <cctype>
#include <cstdint>
#include <cstdlib>
#include <iterator>
#include <limits>
#include <optional>
#include <tuple>
#include <utility>
#include <vector>
#include "AArch64GenAsmMatcher.inc"
Go to the source code of this file.
Classes | |
struct | GenericSetCCInfo |
Helper structure to keep track of ISD::SET_CC operands. More... | |
struct | AArch64SetCCInfo |
Helper structure to keep track of a SET_CC lowered into AArch64 code. More... | |
union | SetCCInfo |
Helper structure to keep track of SetCC information. More... | |
struct | SetCCInfoAndKind |
Helper structure to be able to read SetCC information. More... | |
Macros | |
#define | DEBUG_TYPE "aarch64-lower" |
#define | LCALLNAMES(A, B, N) |
#define | LCALLNAME4(A, B) |
#define | LCALLNAME5(A, B) |
#define | MAKE_CASE(V) |
#define | GET_REGISTER_MATCHER |
Enumerations | |
enum class | PredicateConstraint { Uph , Upl , Upa } |
enum class | ReducedGprConstraint { Uci , Ucj } |
Variables | |
cl::opt< bool > | EnableAArch64ELFLocalDynamicTLSGeneration ("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false)) |
static cl::opt< bool > | EnableOptimizeLogicalImm ("aarch64-enable-logical-imm", cl::Hidden, cl::desc("Enable AArch64 logical imm instruction " "optimization"), cl::init(true)) |
static cl::opt< bool > | EnableCombineMGatherIntrinsics ("aarch64-enable-mgather-combine", cl::Hidden, cl::desc("Combine extends of AArch64 masked " "gather intrinsics"), cl::init(true)) |
static cl::opt< bool > | EnableExtToTBL ("aarch64-enable-ext-to-tbl", cl::Hidden, cl::desc("Combine ext and trunc to TBL"), cl::init(true)) |
static cl::opt< unsigned > | MaxXors ("aarch64-max-xors", cl::init(16), cl::Hidden, cl::desc("Maximum of xors")) |
static const MVT | MVT_CC = MVT::i32 |
Value type used for condition codes. | |
static const MCPhysReg | GPRArgRegs [] |
static const MCPhysReg | FPRArgRegs [] |
#define DEBUG_TYPE "aarch64-lower" |
Definition at line 108 of file AArch64ISelLowering.cpp.
#define GET_REGISTER_MATCHER |
Definition at line 10525 of file AArch64ISelLowering.cpp.
#define MAKE_CASE | ( | V | ) |
|
strong |
Enumerator | |
---|---|
Uph | |
Upl | |
Upa |
Definition at line 10817 of file AArch64ISelLowering.cpp.
|
strong |
Enumerator | |
---|---|
Uci | |
Ucj |
Definition at line 10849 of file AArch64ISelLowering.cpp.
|
static |
Definition at line 4689 of file AArch64ISelLowering.cpp.
References assert(), getExtensionTo64Bits(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::is128BitVector(), and N.
Referenced by skipExtensionForVectorMULL().
|
static |
Definition at line 7532 of file AArch64ISelLowering.cpp.
References llvm::CCState::AllocateStack(), llvm::TargetLowering::CallLoweringInfo::Args, llvm::CallingConv::ARM64EC_Thunk_X64, assert(), llvm::TargetLowering::CallLoweringInfo::CallConv, llvm::AArch64TargetLowering::CCAssignFnForCall(), llvm::TargetLowering::CallLoweringInfo::DAG, llvm::CCValAssign::Full, llvm::SelectionDAG::getDataLayout(), llvm::EVT::getSimpleVT(), llvm::TargetLoweringBase::getValueType(), llvm::AArch64Subtarget::isCallingConvWin64(), llvm::EVT::isSimple(), llvm::TargetLowering::CallLoweringInfo::IsVarArg, llvm::TargetLowering::CallLoweringInfo::Outs, and llvm::SmallVectorBase< Size_T >::size().
Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth of the vector elements.
Definition at line 15311 of file AArch64ISelLowering.cpp.
References llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_ZExtOrSExt(), and llvm::PatternMatch::match().
Referenced by llvm::AArch64TargetLowering::shouldSinkOperands(), and llvm::ARMTargetLowering::shouldSinkOperands().
Check if both Op1 and Op2 are shufflevector extracts of either the lower or upper half of the vector elements.
Definition at line 15255 of file AArch64ISelLowering.cpp.
References llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::Type::getPrimitiveSizeInBits(), llvm::Value::getType(), llvm::ShuffleVectorInst::isExtractSubvectorMask(), isSplatShuffle(), llvm::M1(), llvm::PatternMatch::m_Shuffle(), llvm::PatternMatch::m_Undef(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().
Referenced by llvm::AArch64TargetLowering::shouldSinkOperands().
|
static |
Definition at line 19939 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::all_of(), llvm::ISD::ANY_EXTEND, areLoadedOffsetButOtherwiseSame(), llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), isLoadOrMultipleLoads(), llvm::ISD::SIGN_EXTEND, llvm::SmallVectorBase< Size_T >::size(), Size, llvm::ISD::SUB, llvm::ISD::ZERO_EXTEND, and llvm::zip().
Referenced by areLoadedOffsetButOtherwiseSame(), and performExtBinopLoadFold().
Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
Definition at line 15338 of file AArch64ISelLowering.cpp.
References isOperandOfVmullHighP64().
Referenced by llvm::AArch64TargetLowering::shouldSinkOperands().
Calculates what the pre-extend type is, based on the extension operation node provided by Extend
.
In the case that Extend
is a SIGN_EXTEND or a ZERO_EXTEND, the pre-extend type is pulled directly from the operand, while other extend operations need a bit more inspection to get this information.
Extend | The SDNode from the DAG that represents the extend operation |
Extend
source type, or MVT::Other
if no valid type can be determined Definition at line 17361 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::VTSDNode::getVT(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, and llvm::ISD::ZERO_EXTEND.
Referenced by performBuildShuffleExtendCombine().
|
static |
Returns true if Val
is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.
See CMP;CCMP matching.
CanNegate | Set to true if we can negate the whole sub-tree just by changing the conditions on the SETCC tests. (this means we can call emitConjunctionRec() with Negate==true on this sub-tree) |
MustBeFirst | Set to true if this subtree needs to be negated and we cannot do the negation naturally. We are required to emit the subtree first in this case. |
WillNegate | Is true if are called when the result of this subexpression must be negated. This happens when the outer expression is an OR. We can use this fact to know that we have a double negation (or (or ...) ...) that can be implemented for free. |
Definition at line 3374 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), canEmitConjunction(), llvm::Depth, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::ISD::OR, and llvm::ISD::SETCC.
Referenced by canEmitConjunction(), emitConjunction(), and emitConjunctionRec().
|
static |
Return true if the calling convention is one that we can guarantee TCO for.
Definition at line 7510 of file AArch64ISelLowering.cpp.
References CC, llvm::CallingConv::Fast, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.
Referenced by llvm::AArch64CallLowering::isEligibleForTailCallOptimization(), llvm::AMDGPUCallLowering::isEligibleForTailCallOptimization(), llvm::SITargetLowering::isEligibleForTailCallOptimization(), mayTailCallThisCC(), and shouldGuaranteeTCO().
|
static |
Definition at line 14180 of file AArch64ISelLowering.cpp.
References llvm::Add, llvm::ISD::ADD, assert(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSplatValue(), llvm::SDValue::getValueType(), llvm::EVT::isScalableVT(), and llvm::ISD::SRL.
Referenced by trySimplifySrlAddToRshrnb().
|
static |
Definition at line 3915 of file AArch64ISelLowering.cpp.
References assert(), CC, Cond, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getResNo(), llvm::AArch64CC::HS, and llvm::AArch64CC::LO.
Referenced by lowerADDSUBO_CARRY().
|
static |
changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
Definition at line 3053 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::LE, llvm_unreachable, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.
Referenced by changeFPCCToANDAArch64CC(), and changeVectorFPCCToAArch64CC().
|
static |
Convert a DAG fp condition code to an AArch64 CC.
This differs from changeFPCCToAArch64CC in that it returns cond codes that should be AND'ed instead of OR'ed.
Definition at line 3116 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, assert(), CC, changeFPCCToAArch64CC(), llvm::AArch64CC::LE, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETONE, llvm::ISD::SETUEQ, and llvm::AArch64CC::VC.
Referenced by emitConjunctionRec().
|
static |
changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC
Definition at line 3025 of file AArch64ISelLowering.cpp.
References CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::LE, llvm_unreachable, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::NE, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, and llvm::ISD::SETULT.
Referenced by emitConjunctionRec(), and getAArch64Cmp().
|
static |
changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions.
Fewer operations are available without a real NZCV register, so we have to use less efficient combinations to get the same effect.
Definition at line 3146 of file AArch64ISelLowering.cpp.
References CC, changeFPCCToAArch64CC(), llvm::AArch64CC::GE, llvm::AArch64CC::MI, llvm::ISD::SETO, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, and llvm::ISD::SETUO.
|
static |
Definition at line 22594 of file AArch64ISelLowering.cpp.
References llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::ISD::Constant, llvm::LoadSDNode::getExtensionType(), llvm::MemSDNode::getMemoryVT(), llvm::VTSDNode::getVT(), llvm::ISD::LOAD, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::TargetConstant, and llvm::ISD::ZEXTLOAD.
Referenced by performCONDCombine().
|
static |
Definition at line 7777 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::EVT::getSizeInBits(), and llvm::SDValue::getValueType().
|
static |
Definition at line 20287 of file AArch64ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 22097 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, DL, llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getStore(), llvm::EVT::getStoreSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::EVT::isVector(), llvm::ISD::TRUNCATE, and vectorToScalarBitmask().
Referenced by performSTORECombine().
|
static |
Definition at line 22135 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getUNDEF(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::AArch64Subtarget::isLittleEndian(), and llvm::ISD::TRUNCATE.
Referenced by performSTORECombine().
|
static |
Combines a node carrying the intrinsic aarch64_sve_prf<T>_gather_scalar_offset
into a node that uses aarch64_sve_prfb_gather_uxtw_index
when the scalar offset passed to aarch64_sve_prf<T>_gather_scalar_offset
is not a valid immediate for the sve gather prefetch instruction with vector plus immediate addressing mode.
Definition at line 24147 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), isValidImmForSVEVecImmAddrMode(), N, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20474 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 20455 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 20491 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_VECTOR_ELT, and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 21866 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorVT(), llvm::Or, llvm::ISD::OR, llvm::ISD::SHL, llvm::ISD::TokenFactor, and llvm::ISD::ZERO_EXTEND.
Referenced by performLOADCombine().
|
static |
Definition at line 13213 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::AArch64ISD::FMOV, llvm::ISD::FNEG, llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), llvm::AArch64ISD::MOVI, llvm::AArch64ISD::MOVIedit, llvm::AArch64ISD::MOVImsl, llvm::AArch64ISD::MOVIshift, llvm::AArch64ISD::MVNImsl, llvm::AArch64ISD::MVNIshift, llvm::AArch64ISD::NVCAST, resolveBuildVector(), tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), tryAdvSIMDModImm321s(), tryAdvSIMDModImm64(), tryAdvSIMDModImm8(), tryAdvSIMDModImmFP(), and llvm::APInt::zext().
|
static |
Definition at line 12277 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), Idx, llvm::EVT::is128BitVector(), and WidenVector().
|
static |
Definition at line 26553 of file AArch64ISelLowering.cpp.
References convertToScalableVector(), DL, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getNode(), getPredicateForFixedLengthVector(), llvm::SDValue::getValueType(), llvm::ISD::isBuildVectorAllOnes(), and llvm::AArch64ISD::SETCC_MERGE_ZERO.
|
static |
Definition at line 26503 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and llvm::EVT::isFixedLengthVector().
Referenced by convertLocVTToValVT(), GenerateFixedLengthSVETBL(), getDeinterleaveViaVNSRL(), getVCIXISDNodeWCHAIN(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT(), matchSplatAsGather(), performFP_TO_INTCombine(), and performSVEMulAddSubCombine().
|
static |
Definition at line 20517 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, isAllActivePredicate(), and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 26492 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::ISD::INSERT_SUBVECTOR, and llvm::EVT::isScalableVector().
Referenced by convertFixedMaskToScalableVector(), convertValVTToLocVT(), GenerateFixedLengthSVETBL(), getDeinterleaveViaVNSRL(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerCttzElts(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerScalarInsert(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT(), matchSplatAsGather(), performFP_TO_INTCombine(), performSVEMulAddSubCombine(), processVCIXOperands(), and llvm::RISCVTargetLowering::ReplaceNodeResults().
|
static |
Definition at line 25242 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getTargetConstant(), llvm::DataLayout::isBigEndian(), llvm::SelectionDAG::SplitScalar(), and std::swap().
Referenced by ReplaceCMP_SWAP_128Results(), and ReplaceCMP_SWAP_64Results().
Definition at line 15694 of file AArch64ISelLowering.cpp.
References assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::SmallVectorImpl< T >::clear(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateCall(), llvm::IRBuilderBase::CreateShuffleVector(), llvm::SmallVectorBase< Size_T >::empty(), llvm::Function::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::Instruction::eraseFromParent(), F, llvm::ConstantVector::get(), llvm::FixedVectorType::get(), llvm::Intrinsic::getDeclaration(), llvm::IRBuilderBase::getInt8(), llvm::IRBuilderBase::getInt8Ty(), llvm::Instruction::getModule(), llvm::User::getOperand(), llvm::Value::getType(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::Value::replaceAllUsesWith(), Results, llvm::SmallVectorBase< Size_T >::size(), and llvm::Function::size().
Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().
|
static |
Definition at line 15650 of file AArch64ISelLowering.cpp.
References assert(), llvm::IRBuilderBase::CreateBitCast(), llvm::IRBuilderBase::CreateInsertElement(), llvm::IRBuilderBase::CreateShuffleVector(), llvm::IRBuilderBase::CreateZExt(), llvm::Instruction::eraseFromParent(), llvm::PoisonValue::get(), llvm::VectorType::getElementType(), llvm::IRBuilderBase::getInt8(), llvm::User::getOperand(), llvm::Value::getType(), and llvm::Value::replaceAllUsesWith().
Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().
|
static |
Definition at line 25077 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVectorIdxConstant(), N, Results, and llvm::ISD::SCALAR_TO_VECTOR.
|
static |
Definition at line 3221 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ADDS, llvm::ISD::AND, llvm::AArch64ISD::ANDS, assert(), CC, llvm::AArch64ISD::FCMP, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), isCMN(), llvm::EVT::isFloatingPoint(), llvm::isNullConstant(), LHS, MVT_CC, llvm::SelectionDAG::ReplaceAllUsesWith(), RHS, and llvm::AArch64ISD::SUBS.
Referenced by emitConjunctionRec(), and getAArch64Cmp().
|
static |
can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" which can be implemented as: cmp C ccmp D, inv(CD), CC ccmp A, CA, inv(CD) ccmp B, CB, inv(CA) check for CB flags
A counterexample is "or (and A B) (and C D)" which translates to not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we can only implement 1 of the inner (not) operations, but not both! Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
Definition at line 3326 of file AArch64ISelLowering.cpp.
References assert(), CC, llvm::AArch64ISD::CCMN, llvm::AArch64ISD::CCMP, DL, llvm::AArch64ISD::FCCMP, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SelectionDAG::getSubtarget(), llvm::isNullConstant(), LHS, MVT_CC, RHS, llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::ISD::SUB.
Referenced by emitConjunctionRec().
|
static |
Emit expression as a conjunction (a series of CCMP/CFCMP ops).
In some cases this is even possible with OR operations in the expression. See CMP;CCMP matching.
Definition at line 3548 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, canEmitConjunction(), and emitConjunctionRec().
Referenced by getAArch64Cmp(), LowerBRCOND(), and performANDSETCCCombine().
|
static |
Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
See CMP;CCMP matching. Tries to transform the given i1 producing node Val
to a series compare and conditional compare operations.
OutCC
to the flags that should be tested or returns SDValue() if transformation was not possible. Negate
is true if we want this sub-tree being negated just by changing SETCC conditions. Definition at line 3436 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, llvm::ISD::AND, assert(), canEmitConjunction(), CC, changeFPCCToANDAArch64CC(), changeIntCCToAArch64CC(), DL, emitComparison(), emitConditionalComparison(), emitConjunctionRec(), llvm::AArch64CC::getInvertedCondCode(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::hasOneUse(), LHS, llvm::ISD::OR, RHS, llvm::ISD::SETCC, and std::swap().
Referenced by emitConjunction(), and emitConjunctionRec().
|
static |
Definition at line 3200 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), LHS, RHS, llvm::AArch64ISD::STRICT_FCMP, llvm::AArch64ISD::STRICT_FCMPE, and llvm::ISD::STRICT_FP_EXTEND.
|
static |
Definition at line 14289 of file AArch64ISelLowering.cpp.
References assert(), CC, llvm::AArch64ISD::CMEQ, llvm::AArch64ISD::CMEQz, llvm::AArch64ISD::CMGE, llvm::AArch64ISD::CMGEz, llvm::AArch64ISD::CMGT, llvm::AArch64ISD::CMGTz, llvm::AArch64ISD::CMHI, llvm::AArch64ISD::CMHS, llvm::AArch64ISD::CMLEz, llvm::AArch64ISD::CMLTz, llvm::AArch64CC::EQ, llvm::AArch64ISD::FCMEQ, llvm::AArch64ISD::FCMEQz, llvm::AArch64ISD::FCMGE, llvm::AArch64ISD::FCMGEz, llvm::AArch64ISD::FCMGT, llvm::AArch64ISD::FCMGTz, llvm::AArch64ISD::FCMLEz, llvm::AArch64ISD::FCMLTz, llvm::AArch64CC::GE, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::APInt::isAllOnes(), llvm::BuildVectorSDNode::isConstantSplat(), llvm::EVT::isFloatingPoint(), llvm::AArch64CC::LE, LHS, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, and RHS.
|
static |
Definition at line 22347 of file AArch64ISelLowering.cpp.
References llvm::EVT::changeVectorElementType(), foldIndexIntoBase(), llvm::SDValue::getConstantOperandVal(), llvm::AArch64Subtarget::getMaxSVEVectorSizeInBits(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSplatValue(), llvm::SelectionDAG::getStepVector(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isVectorShrinkable(), N, RHS, llvm::ISD::SHL, llvm::ISD::STEP_VECTOR, llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.
Referenced by performMaskedGatherScatterCombine().
|
static |
Definition at line 19453 of file AArch64ISelLowering.cpp.
References CC, Cond, llvm::AArch64ISD::CSINC, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isNullConstant(), LHS, llvm::AArch64CC::LO, N, and RHS.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23010 of file AArch64ISelLowering.cpp.
References CC, Cond, llvm::AArch64ISD::CSEL, DL, llvm::AArch64CC::EQ, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), isCMP(), llvm::AArch64CC::NE, std::swap(), X, and Y.
Referenced by performCSELCombine().
|
static |
Definition at line 22962 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::BitWidth, CC, llvm::ISD::CTTZ, llvm::AArch64CC::EQ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isNullConstant(), N, llvm::AArch64CC::NE, llvm::AArch64ISD::SUBS, llvm::ISD::TRUNCATE, and X.
Referenced by performCSELCombine().
|
static |
Definition at line 22297 of file AArch64ISelLowering.cpp.
References llvm::Add, llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSplatValue(), llvm::EVT::getVectorElementType(), llvm::EVT::isVector(), llvm::ISD::MUL, llvm::Offset, and llvm::ISD::SHL.
Referenced by findMoreOptimalIndexType().
|
static |
Definition at line 19429 of file AArch64ISelLowering.cpp.
References CC, getCSETCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::AArch64CC::HS, isCMP(), llvm::isNullConstant(), llvm::isOneConstant(), and llvm::AArch64CC::LO.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21828 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValueType(), llvm::ISD::MSTORE, N, llvm::ISD::SIGN_EXTEND, llvm::ISD::STORE, and llvm::ISD::ZERO_EXTEND.
Referenced by performSTORECombine().
|
static |
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X, X, #0.
Definition at line 16928 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::CMGEz, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllOnes(), llvm::EVT::isVector(), N, and llvm::AArch64ISD::VASHR.
Referenced by combineXor(), and performXorCombine().
|
static |
Definition at line 27295 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), convertFromScalableVector(), convertToScalableVector(), llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVScale(), llvm::APInt::getZExtValue(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::ShuffleVectorInst::isSingleSourceMask(), llvm::ISD::MUL, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ArrayRef< T >::size(), llvm::ISD::SPLAT_VECTOR, and llvm::AArch64::SVEBitsPerBlock.
|
static |
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations to build the shuffle.
ID is the perfect-shuffle
Definition at line 12029 of file AArch64ISelLowering.cpp.
References assert(), llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, llvm::AArch64ISD::EXT, llvm::ISD::EXTRACT_VECTOR_ELT, GeneratePerfectShuffle(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getExtFactor(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::ISD::INSERT_VECTOR_ELT, LHS, llvm_unreachable, OP_COPY, OP_VDUP0, OP_VDUP1, OP_VDUP2, OP_VDUP3, OP_VEXT1, OP_VEXT2, OP_VEXT3, OP_VREV, OP_VTRNL, OP_VTRNR, OP_VUZPL, OP_VUZPR, OP_VZIPL, OP_VZIPR, llvm::PerfectShuffleTable, llvm::AArch64ISD::REV16, llvm::AArch64ISD::REV32, llvm::AArch64ISD::REV64, RHS, llvm::AArch64ISD::TRN1, llvm::AArch64ISD::TRN2, llvm::AArch64ISD::UZP1, llvm::AArch64ISD::UZP2, WidenVector(), llvm::AArch64ISD::ZIP1, and llvm::AArch64ISD::ZIP2.
Referenced by GeneratePerfectShuffle(), and LowerVECTOR_SHUFFLE().
|
static |
Definition at line 12190 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::SDValue::isUndef(), isZerosVector(), llvm::Offset, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and std::swap().
|
static |
Definition at line 3596 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, CC, changeIntCCToAArch64CC(), emitComparison(), emitConjunction(), getCmpOperandFoldingProfit(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::ISD::getSetCCSwappedOperands(), llvm::SelectionDAG::getValueType(), llvm::ConstantSDNode::getZExtValue(), INT64_MAX, isCMN(), isLegalArithImmed(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), LHS, MVT_CC, RHS, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SIGN_EXTEND_INREG, std::swap(), UINT64_MAX, and llvm::ISD::ZEXTLOAD.
Referenced by performSetccAddFolding().
|
static |
Definition at line 3727 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ADDS, llvm::AArch64ISD::ANDS, assert(), CC, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::AArch64CC::HS, LHS, llvm_unreachable, llvm::AArch64CC::LO, llvm::Mul, llvm::ISD::MUL, llvm::ISD::MULHS, llvm::ISD::MULHU, MVT_CC, llvm::AArch64CC::NE, RHS, llvm::ISD::SADDO, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMULO, llvm::ISD::SRA, llvm::ISD::SSUBO, llvm::AArch64ISD::SUBS, llvm::ISD::TRUNCATE, llvm::ISD::UADDO, llvm::ISD::UMULO, llvm::ISD::USUBO, llvm::AArch64CC::VS, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerXALUO().
|
static |
Definition at line 25344 of file AArch64ISelLowering.cpp.
References llvm::Acquire, llvm::AcquireRelease, assert(), llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::ATOMIC_LOAD_CLR, llvm::ISD::ATOMIC_LOAD_OR, llvm::ISD::ATOMIC_LOAD_SUB, llvm::ISD::ATOMIC_SWAP, llvm_unreachable, llvm::Monotonic, llvm::Release, and llvm::SequentiallyConsistent.
Referenced by ReplaceATOMIC_LOAD_128Results().
Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
Definition at line 3562 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, and llvm::ISD::SRL.
Referenced by getAArch64Cmp().
Definition at line 4709 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), and llvm::EVT::isFixedLengthVector().
|
static |
Definition at line 26404 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by convertFixedMaskToScalableVector(), GenerateFixedLengthSVETBL(), getContainerForFixedLengthVector(), getDeinterleaveViaVNSRL(), getVCIXISDNodeWCHAIN(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerCttzElts(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), lowerScalarInsert(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT(), matchSplatAsGather(), performFP_TO_INTCombine(), and processVCIXOperands().
|
static |
Definition at line 19411 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, CC, llvm::AArch64ISD::CSEL, llvm::isNullConstant(), llvm::isOneConstant(), and llvm::AArch64CC::NV.
Referenced by foldOverflowCheck().
Definition at line 12264 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, and llvm_unreachable.
Referenced by performDUPCombine().
|
static |
Definition at line 4672 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::EVT::isSimple(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by addRequiredExtensionForVectorMULL(), AddRequiredExtensionForVMULL(), and SkipLoadExtensionForVMULL().
getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction.
Definition at line 11290 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSizeInBits().
Referenced by GeneratePerfectShuffle(), and llvm::AArch64TargetLowering::ReconstructShuffle().
Definition at line 5852 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, and llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO.
Referenced by performGLD1Combine().
Definition at line 6790 of file AArch64ISelLowering.cpp.
References llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::Intrinsic::not_intrinsic.
Referenced by llvm::AMDGPURegisterBankInfo::applyMappingImpl(), collectBitParts(), llvm::SITargetLowering::CollectTargetIntrinsicOperands(), llvm::SITargetLowering::computeKnownBitsForTargetInstr(), FindPreallocatedCall(), llvm::AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic(), llvm::AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects(), llvm::AMDGPURegisterBankInfo::getInstrMapping(), llvm::VPRecipeBuilder::handleReplication(), llvm::SITargetLowering::isCanonicalized(), isFPIntrinsic(), llvm::AArch64TargetLowering::isReassocProfitable(), IsSVECntIntrinsic(), performIntrinsicCombine(), tryConvertSVEWideCompare(), and valueIsKnownNeverF32Denorm().
|
static |
Definition at line 19312 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and llvm::ISD::SUB.
Referenced by performNegCSelCombine().
|
inlinestatic |
Definition at line 187 of file AArch64ISelLowering.cpp.
References llvm_unreachable.
Definition at line 162 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSimpleVT(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by combineSVEReductionInt(), performDupLane128Combine(), and performSVESpliceCombine().
|
static |
Definition at line 26431 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), getPTrue(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::getSVEPredPatternFromNumElements(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by convertFixedMaskToScalableVector(), and getPredicateForVector().
|
static |
Definition at line 26476 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::changeVectorElementType(), DL, getPTrue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isScalableVector(), and llvm::TargetLoweringBase::isTypeLegal().
Referenced by getPredicateForVector().
|
static |
Definition at line 26484 of file AArch64ISelLowering.cpp.
References DL, getPredicateForFixedLengthVector(), getPredicateForScalableVector(), and llvm::EVT::isFixedLengthVector().
Referenced by tryCombineExtendRShTrunc().
|
static |
Definition at line 10829 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorElementType(), llvm::EVT::isScalableVector(), and llvm_unreachable.
Definition at line 202 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::isScalableVector(), and llvm_unreachable.
|
static |
Definition at line 20417 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::ANY_ACTIVE, assert(), CC, Cond, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), getSVEPredicateBitCast(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTypeToTransformTo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::TargetLoweringBase::isTypeLegal(), isZeroingInactiveLanes(), llvm::AArch64CC::NONE_ACTIVE, llvm::AArch64ISD::PTEST, llvm::AArch64ISD::PTEST_ANY, llvm::AArch64ISD::REINTERPRET_CAST, and llvm::Test.
Referenced by performFirstTrueTestVectorCombine(), performIntrinsicCombine(), and performLastTrueTestVectorCombine().
|
inlinestatic |
Definition at line 5080 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), and llvm::AArch64ISD::PTRUE.
Referenced by getPredicateForFixedLengthVector(), getPredicateForScalableVector(), LowerFLDEXP(), optimizeIncrementingWhile(), performFirstTrueTestVectorCombine(), performLastTrueTestVectorCombine(), performMSTORECombine(), and performUnpackCombine().
|
static |
Definition at line 10860 of file AArch64ISelLowering.cpp.
References llvm::EVT::getFixedSizeInBits(), llvm::EVT::isScalarInteger(), and llvm_unreachable.
|
static |
Definition at line 14491 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), and llvm::SDValue::getValueType().
|
static |
Definition at line 23727 of file AArch64ISelLowering.cpp.
References assert(), llvm::BitWidth, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::Log2_32(), llvm::Offset, llvm::ISD::SHL, and llvm::ISD::SPLAT_VECTOR.
Referenced by performGatherLoadCombine(), and performScatterStoreCombine().
|
static |
Helper function to create 'CSET', which is equivalent to 'CSINC <Wd>, WZR, WZR, invert(<cond>)'.
Definition at line 10900 of file AArch64ISelLowering.cpp.
References CC, llvm::AArch64ISD::CSINC, DL, llvm::SelectionDAG::getConstant(), and llvm::SelectionDAG::getNode().
Referenced by combineAnd(), combineCMov(), combineM68kSetCC(), combineOr(), combinePredicateReduction(), combineSetCC(), combineSubSetcc(), combineVectorSizedSetCCEquality(), combineX86SetCC(), foldXor1SetCC(), LowerADDSUBO_CARRY(), llvm::X86TargetLowering::LowerAsmOutputForConstraint(), LowerCMP_SWAP(), LowerINTRINSIC_W_CHAIN(), LowerPARITY(), LowerXALUO(), and llvm::X86TargetLowering::ReplaceNodeResults().
Definition at line 5875 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1S_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO, and llvm_unreachable.
Referenced by performGLD1Combine().
|
static |
Definition at line 7858 of file AArch64ISelLowering.cpp.
References llvm::AArch64SME::Always, llvm::SMEAttrs::hasNonStreamingInterface(), llvm::SMEAttrs::hasStreamingBody(), llvm::SMEAttrs::hasStreamingCompatibleInterface(), llvm::SMEAttrs::hasStreamingInterface(), llvm::AArch64SME::IfCallerIsNonStreaming, llvm::AArch64SME::IfCallerIsStreaming, and llvm_unreachable.
|
static |
Definition at line 16014 of file AArch64ISelLowering.cpp.
References assert(), and llvm::Intrinsic::getDeclaration().
Referenced by llvm::AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(), and llvm::AArch64TargetLowering::lowerInterleavedLoad().
|
static |
Definition at line 16030 of file AArch64ISelLowering.cpp.
References assert(), and llvm::Intrinsic::getDeclaration().
Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore(), and llvm::AArch64TargetLowering::lowerInterleaveIntrinsicToStore().
|
static |
Definition at line 15986 of file AArch64ISelLowering.cpp.
References llvm::ScalableVectorType::get(), llvm::Type::getBFloatTy(), llvm::Type::getContext(), llvm::Type::getDoubleTy(), llvm::VectorType::getElementType(), llvm::Type::getFloatTy(), llvm::Type::getHalfTy(), llvm::Type::getInt16Ty(), llvm::Type::getInt32Ty(), llvm::Type::getInt64Ty(), llvm::Type::getInt8Ty(), and llvm_unreachable.
Referenced by llvm::AArch64TargetLowering::lowerInterleavedLoad(), and llvm::AArch64TargetLowering::lowerInterleavedStore().
Definition at line 20954 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getSimpleVT(), llvm::EVT::isSimple(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by performGatherLoadCombine(), performLD1Combine(), performScatterStoreCombine(), and performST1Combine().
|
static |
Definition at line 5126 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::EVT::bitsGT(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), isZeroingInactiveLanes(), and llvm::AArch64ISD::REINTERPRET_CAST.
Referenced by getPTest().
|
static |
Definition at line 23330 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::CallingConv::C, llvm::DWARFExpression::Operation::getNumOperands(), getTestBitOperand(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::XOR.
Referenced by getTestBitOperand(), and performTBZCombine().
|
static |
Definition at line 14499 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), getVectorBitwiseReduce(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::EVT::isFixedLengthVector(), llvm::EVT::isPow2VectorType(), llvm_unreachable, llvm::Lo, llvm::ISD::OR, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::SplitVector(), llvm::ISD::SRL, llvm::ISD::VECREDUCE_ADD, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_OR, llvm::ISD::VECREDUCE_UMAX, llvm::ISD::VECREDUCE_UMIN, llvm::ISD::VECREDUCE_XOR, and llvm::ISD::XOR.
Referenced by getVectorBitwiseReduce(), and performVecReduceBitwiseCombine().
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value.
Definition at line 14114 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::APInt::getSExtValue(), and llvm::BuildVectorSDNode::isConstantSplat().
Referenced by isVShiftLImm(), and isVShiftRImm().
bool hasNearbyPairedStore | ( | Iter | It, |
Iter | End, | ||
Value * | Ptr, | ||
const DataLayout & | DL | ||
) |
Definition at line 16179 of file AArch64ISelLowering.cpp.
References DL, End, Ptr, and llvm::APInt::sextOrTrunc().
Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore().
Definition at line 18462 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::FADD, and llvm::ISD::STRICT_FADD.
Referenced by performExtractVectorEltCombine().
|
static |
Definition at line 4790 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isSignExtended(), N, and llvm::ISD::SUB.
Referenced by LowerMUL(), and selectUmullSmull().
|
static |
Definition at line 4801 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isZeroExtended(), N, and llvm::ISD::SUB.
Referenced by LowerMUL(), and selectUmullSmull().
|
static |
Definition at line 12987 of file AArch64ISelLowering.cpp.
References llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::getNumElementsFromSVEPredPattern(), llvm::SelectionDAG::getSubtarget(), llvm::ISD::isConstantSplatVectorAllOnes(), N, llvm::AArch64ISD::PTRUE, llvm::AArch64ISD::REINTERPRET_CAST, and llvm::AArch64::SVEBitsPerBlock.
Referenced by convertMergedOpToPredOp(), instCombineSVELD1(), instCombineSVESel(), instCombineSVESrshl(), instCombineSVEST1(), performSetccMergeZeroCombine(), performSVEAndCombine(), performVSelectCombine(), and tryLowerToSLI().
Definition at line 12962 of file AArch64ISelLowering.cpp.
References llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), and llvm::ConstantSDNode::getZExtValue().
Referenced by isPow2Splat().
Definition at line 12979 of file AArch64ISelLowering.cpp.
References llvm::ISD::isConstantSplatVectorAllZeros(), N, and llvm::AArch64ISD::REINTERPRET_CAST.
Referenced by performVSelectCombine().
Definition at line 20839 of file AArch64ISelLowering.cpp.
References llvm::ISD::isConstantSplatVectorAllZeros(), llvm::ISD::LOAD, llvm::ISD::MLOAD, and N.
Referenced by performSignExtendSetCCCombine().
|
static |
Definition at line 3195 of file AArch64ISelLowering.cpp.
References CC, llvm::isNullConstant(), llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::ISD::SUB.
Referenced by emitComparison(), and getAArch64Cmp().
Definition at line 19404 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::SUBS.
Referenced by foldCSELOfCSEL(), and foldOverflowCheck().
Definition at line 11976 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), I, and llvm::Offset.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal(), and tryFormConcatFromShuffle().
Definition at line 18173 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::DUP, llvm::EVT::getSimpleVT(), llvm::EVT::getVectorElementType(), llvm::EVT::isSimple(), N, llvm::MVT::SimpleTy, and llvm::ISD::SPLAT_VECTOR.
|
static |
Definition at line 22697 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::NV, llvm::AArch64CC::PL, llvm::ISD::SEXTLOAD, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.
Referenced by performCONDCombine().
Definition at line 19027 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_SUBVECTOR, and N.
Referenced by performAddSubLongCombine(), tryCombineLongOpWithDup(), and tryCombineMULLWithUZP1().
|
static |
Definition at line 4722 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, llvm::EVT::getScalarSizeInBits(), llvm::isIntN(), isSigned(), llvm::isUIntN(), and N.
Referenced by isSignExtended(), isZeroExtended(), and selectUmullSmull().
Definition at line 19656 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ConstantSDNode::getZExtValue(), llvm::ISD::isExtOpcode(), N, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, and llvm::ISD::SRL.
Referenced by performAddCombineSubShift().
Definition at line 11844 of file AArch64ISelLowering.cpp.
References llvm::find_if(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), and llvm::APInt::logBase2().
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
Definition at line 21363 of file AArch64ISelLowering.cpp.
References llvm::SDNode::getValueType(), N, and llvm::AArch64ISD::UZP1.
Referenced by performUzpCombine().
Definition at line 22128 of file AArch64ISelLowering.cpp.
Referenced by performMSTORECombine(), and performSTORECombine().
|
static |
Definition at line 11937 of file AArch64ISelLowering.cpp.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
Definition at line 2019 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, and N.
Definition at line 24168 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ANDV_PRED, llvm::AArch64ISD::EORV_PRED, llvm::AArch64ISD::FADDA_PRED, llvm::AArch64ISD::FADDV_PRED, llvm::AArch64ISD::FMAXNMV_PRED, llvm::AArch64ISD::FMAXV_PRED, llvm::AArch64ISD::FMINNMV_PRED, llvm::AArch64ISD::FMINV_PRED, llvm::AArch64ISD::ORV_PRED, llvm::AArch64ISD::SADDV_PRED, llvm::AArch64ISD::SMAXV_PRED, llvm::AArch64ISD::SMINV_PRED, llvm::AArch64ISD::UADDV_PRED, llvm::AArch64ISD::UMAXV_PRED, and llvm::AArch64ISD::UMINV_PRED.
Referenced by removeRedundantInsertVectorElt().
Definition at line 3177 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, llvm::dbgs(), and LLVM_DEBUG.
Referenced by getAArch64Cmp().
|
static |
Definition at line 19868 of file AArch64ISelLowering.cpp.
References B, llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::SDNode::hasOneUse(), I, llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::VECTOR_SHUFFLE.
Referenced by areLoadedOffsetButOtherwiseSame(), and performExtBinopLoadFold().
Definition at line 233 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ABS_MERGE_PASSTHRU, llvm::AArch64ISD::BITREVERSE_MERGE_PASSTHRU, llvm::AArch64ISD::BSWAP_MERGE_PASSTHRU, llvm::AArch64ISD::CTLZ_MERGE_PASSTHRU, llvm::AArch64ISD::CTPOP_MERGE_PASSTHRU, llvm::AArch64ISD::DUP_MERGE_PASSTHRU, llvm::AArch64ISD::FABS_MERGE_PASSTHRU, llvm::AArch64ISD::FCEIL_MERGE_PASSTHRU, llvm::AArch64ISD::FCVTZS_MERGE_PASSTHRU, llvm::AArch64ISD::FCVTZU_MERGE_PASSTHRU, llvm::AArch64ISD::FFLOOR_MERGE_PASSTHRU, llvm::AArch64ISD::FNEARBYINT_MERGE_PASSTHRU, llvm::AArch64ISD::FNEG_MERGE_PASSTHRU, llvm::AArch64ISD::FP_EXTEND_MERGE_PASSTHRU, llvm::AArch64ISD::FP_ROUND_MERGE_PASSTHRU, llvm::AArch64ISD::FRECPX_MERGE_PASSTHRU, llvm::AArch64ISD::FRINT_MERGE_PASSTHRU, llvm::AArch64ISD::FROUND_MERGE_PASSTHRU, llvm::AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU, llvm::AArch64ISD::FSQRT_MERGE_PASSTHRU, llvm::AArch64ISD::FTRUNC_MERGE_PASSTHRU, llvm::AArch64ISD::NEG_MERGE_PASSTHRU, llvm::AArch64ISD::REVD_MERGE_PASSTHRU, llvm::AArch64ISD::REVH_MERGE_PASSTHRU, llvm::AArch64ISD::REVW_MERGE_PASSTHRU, llvm::AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, llvm::AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU, llvm::AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU, and llvm::AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU.
Definition at line 19308 of file AArch64ISelLowering.cpp.
References llvm::isNullConstant(), and llvm::ISD::SUB.
Referenced by performNegCSelCombine().
Definition at line 2030 of file AArch64ISelLowering.cpp.
References isIntImmediate(), and N.
Check if Op could be used with vmull_high_p64 intrinsic.
Definition at line 15327 of file AArch64ISelLowering.cpp.
References llvm::Value::getType(), llvm::ConstantInt::getValue(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_ExtractElt(), llvm::PatternMatch::m_Value(), and llvm::PatternMatch::match().
Referenced by areOperandsOfVmullHighP64().
|
static |
Definition at line 9668 of file AArch64ISelLowering.cpp.
References isOrXorChain(), MaxXors, N, llvm::ISD::OR, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by isOrXorChain(), and performOrXorChainCombine().
|
inlinestatic |
Returns true if VT's elements occupy the lowest bit positions of its associated register class without any intervening space.
For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the same register class, but only nxv8f16 can be treated as a packed vector.
Definition at line 224 of file AArch64ISelLowering.cpp.
References assert(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), and llvm::AArch64::SVEBitsPerBlock.
Definition at line 6929 of file AArch64ISelLowering.cpp.
References llvm::EVT::isFixedLengthVector(), llvm::EVT::isFloatingPoint(), and llvm::EVT::isScalableVector().
Definition at line 13992 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::AArch64ISD::DUP, isAllConstantBuildVector(), llvm::isPowerOf2_64(), and llvm::ISD::SPLAT_VECTOR.
Definition at line 18477 of file AArch64ISelLowering.cpp.
References llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::ISD::SETCC.
Referenced by performFirstTrueTestVectorCombine().
|
static |
Check whether or not Op
is a SET_CC operation, either a generic or an AArch64 lowered one.
SetCCInfo
is filled accordingly.
Definition at line 19071 of file AArch64ISelLowering.cpp.
References SetCCInfo::AArch64, GenericSetCCInfo::CC, AArch64SetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::AArch64ISD::CSEL, SetCCInfo::Generic, llvm::AArch64CC::getInvertedCondCode(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, llvm::ISD::SETCC, and std::swap().
Referenced by isSetCCOrZExtSetCC().
|
static |
Definition at line 19114 of file AArch64ISelLowering.cpp.
References Info, isSetCC(), and llvm::ISD::ZERO_EXTEND.
Referenced by performSetccAddFolding().
|
static |
Definition at line 4778 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::SIGN_EXTEND.
Referenced by llvm::PPCTargetLowering::EmitPartwordAtomicBinary(), isAddSubSExt(), LowerMUL(), performMulCombine(), and selectUmullSmull().
Definition at line 11688 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements().
Definition at line 15247 of file AArch64ISelLowering.cpp.
References llvm::all_equal().
Referenced by areExtractShuffleVectors(), and llvm::AArch64TargetLowering::shouldSinkOperands().
Definition at line 17334 of file AArch64ISelLowering.cpp.
References getIntrinsicID(), and llvm::SDValue::getNode().
Referenced by performMulCombine().
isTRN_v_undef_Mask - Special case of isTRNMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
Mask is e.g., <0, 0, 2, 2> instead of <0, 4, 2, 6>.
Definition at line 11924 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements().
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
isUZP_v_undef_Mask - Special case of isUZPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
Mask is e.g., <0, 2, 0, 2> instead of <0, 2, 4, 6>,
Definition at line 11905 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), and Idx.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
Check if the value of Offset
represents a valid immediate for the SVE gather load/prefetch and scatter store instructiona with vector base and immediate offset addressing mode:
[<Zn>.[S|D]{, #<imm>}]
where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
Definition at line 23765 of file AArch64ISelLowering.cpp.
References llvm::ConstantSDNode::getZExtValue(), isValidImmForSVEVecImmAddrMode(), and llvm::Offset.
|
inlinestatic |
Check if the value of OffsetInBytes
can be used as an immediate for the gather load/prefetch and scatter store instructions with vector base and immediate offset addressing mode:
[<Zn>.[S|D]{, #<imm>}]
where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
Definition at line 23745 of file AArch64ISelLowering.cpp.
Referenced by combineSVEPrefetchVecBaseImmOff(), isValidImmForSVEVecImmAddrMode(), performGatherLoadCombine(), and performScatterStoreCombine().
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation.
That value must be in the range: 0 <= Value < ElementBits for a left shift; or 0 <= Value <= ElementBits for a long left shift.
Definition at line 14134 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().
Referenced by LowerShift(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), and PerformShiftCombine().
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation.
The value must be in the range: 1 <= Value <= ElementBits for a right shift; or
Definition at line 14145 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().
Referenced by LowerShift(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), and PerformShiftCombine().
|
static |
Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vector lane type.
If that is the case the function returns true and writes the value of the DUP instruction lane operand into DupLaneOp
Definition at line 11771 of file AArch64ISelLowering.cpp.
References assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), BlockSize, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), and I.
|
static |
Definition at line 12339 of file AArch64ISelLowering.cpp.
References assert(), llvm::SmallVectorImpl< T >::clear(), llvm::EVT::getVectorNumElements(), llvm::M0(), llvm::M1(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallVectorBase< Size_T >::size().
Referenced by tryWidenMaskForShuffle().
|
static |
Definition at line 4784 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::ZERO_EXTEND.
Referenced by isAddSubZExt(), LowerMUL(), performMulCombine(), and selectUmullSmull().
Definition at line 271 of file AArch64ISelLowering.cpp.
References llvm::ISD::INTRINSIC_WO_CHAIN, llvm::AArch64ISD::PTRUE, llvm::AArch64ISD::SETCC_MERGE_ZERO, and llvm::ISD::SPLAT_VECTOR.
Referenced by getPTest(), and getSVEPredicateBitCast().
isZerosVector - Check whether SDNode N is a zero-filled vector.
Definition at line 3008 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::AArch64ISD::DUP, llvm::ISD::isConstantSplatVectorAllZeros(), llvm::isNullConstant(), llvm::isNullFPConstant(), and N.
Referenced by GenerateTBL(), performAddDotCombine(), performSetccMergeZeroCombine(), performSetCCPunpkCombine(), and performUnpackCombine().
isZIP_v_undef_Mask - Special case of isZIPMask for canonical form of "vector_shuffle v, v", i.e., "vector_shuffle v, undef".
Mask is e.g., <0, 0, 1, 1> instead of <0, 4, 1, 5>.
Definition at line 11886 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), and Idx.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
|
static |
Legalize the gather prefetch (scalar + vector addressing mode) when the offset vector is an unpacked 32-bit scalable vector.
The other cases (Offset != nxv2i32) do not need legalization.
Definition at line 24124 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), N, and llvm::Offset.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
Definition at line 9228 of file AArch64ISelLowering.cpp.
References llvm::EVT::getFixedSizeInBits(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::SIGN_EXTEND_INREG.
|
static |
Definition at line 3938 of file AArch64ISelLowering.cpp.
References carryFlagToValue(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::ISD::MERGE_VALUES, overflowFlagToValue(), llvm::AArch64ISD::SBCS, and valueToCarryFlag().
Referenced by llvm::AArch64TargetLowering::LowerOperation().
|
static |
Definition at line 6319 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::BRCOND, CC, Cond, emitConjunction(), llvm::SelectionDAG::getConstant(), and llvm::SelectionDAG::getNode().
Referenced by llvm::AArch64TargetLowering::LowerOperation().
|
static |
Definition at line 6359 of file AArch64ISelLowering.cpp.
References llvm::EVT::changeVectorElementType(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), getPTrue(), llvm::SelectionDAG::getUNDEF(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::ISD::SIGN_EXTEND, and X.
Referenced by llvm::AArch64TargetLowering::LowerOperation().
|
static |
Definition at line 6337 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getConstant(), llvm::MVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), and llvm::SDValue::getValueType().
Referenced by llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().
|
static |
Definition at line 3995 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), and llvm::AArch64ISD::PREFETCH.
Referenced by llvm::AArch64TargetLowering::LowerOperation(), llvm::ARMTargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().
SDValue LowerSMELdrStr | ( | SDValue | N, |
SelectionDAG & | DAG, | ||
bool | IsLoad | ||
) |
Definition at line 5218 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::sampleprof::Base, llvm::CallingConv::C, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::Mul, llvm::ISD::MUL, N, llvm::AArch64ISD::RDSVL, llvm::ISD::SIGN_EXTEND, llvm::AArch64ISD::SME_ZA_LDR, and llvm::AArch64ISD::SME_ZA_STR.
|
static |
Definition at line 20313 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::AArch64ISD::DUP_MERGE_PASSTHRU, llvm::SelectionDAG::getNode(), and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 20327 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::AArch64ISD::EXT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::ElementCount::getScalable(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::EVT::isScalableVector(), llvm::ISD::MUL, N, and llvm::AArch64::SVEBitsPerBlock.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 20297 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::sampleprof::Base, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStepVector(), llvm::SDValue::getValueType(), llvm::Mul, llvm::ISD::MUL, N, and llvm::ISD::SPLAT_VECTOR.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 6104 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStore(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isVector(), and llvm::ISD::TRUNCATE.
|
static |
Definition at line 3964 of file AArch64ISelLowering.cpp.
References CC, llvm::AArch64ISD::CSEL, getAArch64XALUOOp(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getVTList(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::ISD::MERGE_VALUES.
Referenced by LowerMULO(), llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().
|
static |
Return true if we might ever do TCO for calls with this calling convention.
Definition at line 7516 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::AArch64_SVE_VectorCall, llvm::CallingConv::C, CC, llvm::CallingConv::Fast, llvm::CallingConv::PreserveAll, llvm::CallingConv::PreserveMost, llvm::CallingConv::Swift, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.
Referenced by llvm::AArch64CallLowering::isEligibleForTailCallOptimization(), llvm::AMDGPUCallLowering::isEligibleForTailCallOptimization(), and llvm::SITargetLowering::isEligibleForTailCallOptimization().
|
static |
Definition at line 13182 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::APInt::getZExtValue(), llvm::EVT::isFloatingPoint(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
|
static |
Definition at line 5088 of file AArch64ISelLowering.cpp.
References llvm::APInt::getBitWidth(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), getPTrue(), llvm::SelectionDAG::getSubtarget(), llvm::getSVEPredPatternFromNumElements(), llvm::APInt::getZExtValue(), llvm::APInt::sadd_ov(), llvm::APInt::uadd_ov(), X, and Y.
|
static |
Definition at line 2036 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::TargetLoweringOpt::CombineTo(), llvm::TargetLowering::TargetLoweringOpt::DAG, DL, llvm::AArch64_AM::encodeLogicalImmediate(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::APInt::getZExtValue(), llvm::Hi, llvm::AArch64_AM::isLogicalImmediate(), llvm::isShiftedMask_64(), and Size.
Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant().
|
static |
Definition at line 3927 of file AArch64ISelLowering.cpp.
References assert(), CC, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getResNo(), and llvm::AArch64CC::VS.
Referenced by lowerADDSUBO_CARRY().
|
static |
Definition at line 10876 of file AArch64ISelLowering.cpp.
References llvm::StringSwitch< T, R >::Case(), Cond, llvm::StringSwitch< T, R >::Default(), llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.
Referenced by llvm::X86TargetLowering::getConstraintType(), llvm::X86TargetLowering::getRegForInlineAsmConstraint(), and llvm::X86TargetLowering::LowerAsmOutputForConstraint().
|
static |
Definition at line 10820 of file AArch64ISelLowering.cpp.
References llvm::StringSwitch< T, R >::Case(), and llvm::StringSwitch< T, R >::Default().
|
static |
Definition at line 10852 of file AArch64ISelLowering.cpp.
References llvm::StringSwitch< T, R >::Case(), and llvm::StringSwitch< T, R >::Default().
|
static |
Definition at line 19709 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), llvm::Value::hasOneUse(), isOpcWithIntImmediate(), LHS, N, performAddCombineSubShift(), RHS, and llvm::ISD::SHL.
Referenced by performAddSubCombine().
|
static |
Definition at line 19681 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), isExtendOrShiftOperand(), N, llvm::ISD::SUB, and Y.
Referenced by performAddCombineForShiftedOperands().
|
static |
Perform the scalar expression combine in the form of: CSEL(c, 1, cc) + b => CSINC(b+c, b, cc) CSNEG(c, -1, cc) + b => CSINC(b+c, b, cc)
Definition at line 19213 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::CallingConv::C, llvm::AArch64ISD::CSEL, llvm::AArch64ISD::CSINC, llvm::AArch64ISD::CSNEG, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::Value::hasOneUse(), llvm::ConstantSDNode::isAllOnes(), llvm::TargetLoweringBase::isLegalAddImmediate(), llvm::ConstantSDNode::isOne(), llvm::EVT::isScalarInteger(), LHS, N, RHS, and std::swap().
Referenced by performAddSubCombine().
|
static |
Definition at line 19286 of file AArch64ISelLowering.cpp.
References A, llvm::ISD::ADD, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), isZerosVector(), N, llvm::AArch64ISD::SDOT, std::swap(), and llvm::AArch64ISD::UDOT.
Referenced by performAddSubCombine().
|
static |
Definition at line 20112 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, N, performAddCombineForShiftedOperands(), performAddCSelIntoCSinc(), performAddDotCombine(), performAddSubIntoVectorOp(), performAddSubLongCombine(), performAddUADDVCombine(), performExtBinopLoadFold(), performNegCSelCombine(), performSubAddMULCombine(), performSVEMulAddSubCombine(), and performVectorAddSubExtCombine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19835 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::LOAD, N, and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by performAddSubCombine().
|
static |
Definition at line 19362 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getNode(), llvm::MVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, performSetccAddFolding(), RHS, llvm::ISD::SIGN_EXTEND, tryExtendDUPToExtractHigh(), and llvm::ISD::ZERO_EXTEND.
Referenced by performAddSubCombine().
|
static |
Definition at line 19174 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::isScalarInteger(), LHS, N, RHS, and llvm::AArch64ISD::UADDV.
Referenced by performAddSubCombine().
|
static |
Definition at line 18363 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::BICi, llvm::SelectionDAG::computeKnownBits(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::APInt::getBitWidth(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), I, llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), LHS, N, performANDORCSELCombine(), performANDSETCCCombine(), performSVEAndCombine(), resolveBuildVector(), RHS, tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), llvm::KnownBits::Zero, and llvm::APInt::zext().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().
|
static |
Definition at line 18087 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::AArch64ISD::CCMN, llvm::AArch64ISD::CCMP, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::isNullConstant(), llvm::isOneConstant(), MVT_CC, N, llvm::AArch64ISD::SUBS, and std::swap().
Referenced by performANDCombine(), and performORCombine().
|
static |
Definition at line 18324 of file AArch64ISelLowering.cpp.
References CC, llvm::AArch64ISD::CSINC, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, emitConjunction(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, llvm::ISD::SELECT, and llvm::ISD::SETCC.
Referenced by performANDCombine().
|
static |
Definition at line 22900 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ADDS, assert(), llvm::AArch64ISD::CBNZ, llvm::AArch64ISD::CBZ, CC, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::AArch64CC::EQ, llvm::SDNode::getAsZExtVal(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::Function::hasFnAttribute(), llvm::isNullConstant(), LHS, N, llvm::AArch64CC::NE, performCONDCombine(), RHS, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::AArch64ISD::SUBS, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24295 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::isScalableVector(), N, and llvm::ISD::OR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Combines a buildvector(sext/zext) or shuffle(sext/zext, undef) node pattern into sext/zext(buildvector) or sext/zext(shuffle) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
Definition at line 17399 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::ISD::BUILD_VECTOR, calculatePreExtendType(), llvm::EVT::changeVectorElementType(), DL, llvm::drop_begin(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::isUndef(), llvm::SDNode::ops(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::VECTOR_SHUFFLE, and llvm::ISD::ZERO_EXTEND.
Referenced by performMulVectorExtendCombine().
|
static |
Definition at line 19504 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::EVT::changeVectorElementType(), Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::AArch64ISD::FCVTXN, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorMinNumElements(), llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::ISD::UNDEF.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18631 of file AArch64ISelLowering.cpp.
References llvm::Add, llvm::ISD::ADD, llvm::all_of(), assert(), llvm::ISD::BITCAST, CC, llvm::ISD::CONCAT_VECTORS, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dbgs(), llvm::AArch64ISD::DUP, llvm::AArch64ISD::DUPLANE64, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SDNode::hasOneUse(), llvm::EVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isBinOp(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::SDNode::isOnlyUserOf(), llvm::EVT::isScalableVector(), llvm::SDValue::isUndef(), llvm::MVT::isVector(), LLVM_DEBUG, llvm::AArch64ISD::MOVIshift, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::TRUNCATE, llvm::AArch64ISD::VLSHR, WidenVector(), X, llvm::ISD::XOR, Y, llvm::AArch64ISD::ZIP1, and llvm::AArch64ISD::ZIP2.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 22821 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, CC, checkValueWidth(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDNode::hasOneUse(), isEquivalentMaskless(), N, performSubsToAndsCombine(), llvm::SelectionDAG::ReplaceAllUsesWith(), and llvm::AArch64ISD::SUBS.
Referenced by performBRCONDCombine(), and performCSELCombine().
|
static |
Definition at line 23064 of file AArch64ISelLowering.cpp.
References foldCSELOfCSEL(), foldCSELofCTTZ(), N, and performCONDCombine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23714 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITREVERSE, llvm::ISD::CTTZ, DL, llvm::SelectionDAG::getNode(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23608 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::AArch64ISD::DUP, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), getDUPLANEOp(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), N, and performPostLD1Combine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24315 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, DL, llvm::AArch64ISD::DUPLANE128, llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19994 of file AArch64ISelLowering.cpp.
References areLoadedOffsetButOtherwiseSame(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::Hi, llvm::ISD::isConstantSplatVector(), llvm::ISD::isExtOpcode(), llvm::EVT::isFixedLengthVector(), isLoadOrMultipleLoads(), llvm::TargetLoweringBase::isTypeLegal(), llvm::Lo, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::SDNode::op_values(), llvm::Other, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SHL, llvm::SmallVectorBase< Size_T >::size(), llvm::ISD::SUB, std::swap(), and llvm::zip().
Referenced by performAddSubCombine().
|
static |
Definition at line 20885 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, llvm::Intrinsic::not_intrinsic, performSignExtendSetCCCombine(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, tryCombineLongOpWithDup(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18859 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isScalableVector(), N, and llvm::ISD::SPLAT_VECTOR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18562 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::AArch64ISD::DUP, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), hasPairwiseAdd(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::SDNode::isStrictFPOpcode(), N, llvm::Other, performFirstTrueTestVectorCombine(), performLastTrueTestVectorCombine(), and llvm::SelectionDAG::ReplaceAllUsesOfValueWith().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18426 of file AArch64ISelLowering.cpp.
References A, B, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::FADD, llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, LHS, N, RHS, and llvm::SDNode::setFlags().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Fold a floating-point divide by power of two into fixed-point to floating-point conversion.
Definition at line 17912 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, DL, llvm::SelectionDAG::getConstant(), llvm::BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(), llvm::SelectionDAG::getNode(), llvm::MVT::getSizeInBits(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, llvm::ISD::SIGN_EXTEND, llvm::ISD::SINT_TO_FP, llvm::ISD::UINT_TO_FP, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18498 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::AArch64CC::FIRST_ACTIVE, getPTest(), getPTrue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::isNullConstant(), isPredicateCCSettingOp(), llvm::EVT::isScalableVector(), and N.
Referenced by performExtractVectorEltCombine().
|
static |
Definition at line 23222 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, Generic, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getVTList(), LHS, N, and RHS.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24257 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::EXTLOAD, llvm::ISD::FP_ROUND, llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getExtLoad(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MemSDNode::getMemOperand(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isNormalLoad(), N, and llvm::AArch64Subtarget::useSVEForFixedLengthVectors().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.
Definition at line 17841 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, DL, llvm::ISD::FMUL, llvm::ISD::FP_TO_SINT, llvm::ISD::FP_TO_SINT_SAT, llvm::ISD::FP_TO_UINT_SAT, llvm::SelectionDAG::getConstant(), llvm::BuildVectorSDNode::getConstantFPSplatPow2ToLog2Int(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::AArch64Subtarget::isNeonAvailable(), llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23885 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, llvm::ISD::BITCAST, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1Q_INDEX_MERGE_ZERO, llvm::AArch64ISD::GLD1Q_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_INDEX_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_MERGE_ZERO, llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::EVT::isScalableVector(), isValidImmForSVEVecImmAddrMode(), N, llvm::Offset, llvm::AArch64::SVEBitsPerBlock, std::swap(), and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21591 of file AArch64ISelLowering.cpp.
References assert(), llvm::sampleprof::Base, DL, getGatherVecOpcode(), llvm::SelectionDAG::getNode(), getSignExtendedGatherOpcode(), llvm::EVT::getVectorElementType(), llvm::VTSDNode::getVT(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1S_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SCALED_MERGE_ZERO, N, llvm::Offset, Scaled, llvm::AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU, Signed, and llvm::AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23663 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::CallingConv::C, llvm::AArch64Subtarget::ClassifyGlobalReference(), DL, llvm::SelectionDAG::getConstant(), llvm::Module::getDataLayout(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getGlobalAddress(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::GlobalValue::getParent(), llvm::DataLayout::getTypeAllocSize(), llvm::GlobalValue::getValueType(), llvm::AArch64II::MO_NO_FLAG, N, llvm::Offset, llvm::ISD::SUB, TM, and llvm::SDNode::uses().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18882 of file AArch64ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::Lo, and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24228 of file AArch64ISelLowering.cpp.
References N, performPostLD1Combine(), and removeRedundantInsertVectorElt().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20538 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ADD, llvm::ISD::AND, llvm::AArch64ISD::ANDV_PRED, llvm::AArch64CC::ANY_ACTIVE, llvm::AArch64ISD::BIC, combineAcrossLanesIntrinsic(), combineSVEReductionFP(), combineSVEReductionInt(), combineSVEReductionOrderedFP(), convertMergedOpToPredOp(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::AArch64ISD::EORV_PRED, llvm::AArch64ISD::FADD_PRED, llvm::AArch64ISD::FADDA_PRED, llvm::AArch64ISD::FADDV_PRED, llvm::AArch64ISD::FDIV_PRED, llvm::AArch64CC::FIRST_ACTIVE, llvm::AArch64ISD::FMA_PRED, llvm::AArch64ISD::FMAX_PRED, llvm::ISD::FMAXIMUM, llvm::AArch64ISD::FMAXNM_PRED, llvm::AArch64ISD::FMAXNMV_PRED, llvm::ISD::FMAXNUM, llvm::AArch64ISD::FMAXV_PRED, llvm::AArch64ISD::FMIN_PRED, llvm::ISD::FMINIMUM, llvm::AArch64ISD::FMINNM_PRED, llvm::AArch64ISD::FMINNMV_PRED, llvm::ISD::FMINNUM, llvm::AArch64ISD::FMINV_PRED, llvm::AArch64ISD::FMUL_PRED, llvm::AArch64ISD::FSUB_PRED, llvm::SelectionDAG::getCondCode(), getIntrinsicID(), llvm::SelectionDAG::getNode(), getPTest(), llvm::AArch64CC::LAST_ACTIVE, LowerSVEIntrinsicDUP(), LowerSVEIntrinsicEXT(), LowerSVEIntrinsicIndex(), llvm::AArch64ISD::MUL_PRED, llvm::AArch64ISD::MULHS_PRED, llvm::AArch64ISD::MULHU_PRED, N, llvm::ISD::OR, llvm::AArch64ISD::ORV_PRED, llvm::AArch64ISD::PMULL, llvm::ISD::SADDSAT, llvm::AArch64ISD::SADDV, llvm::AArch64ISD::SADDV_PRED, llvm::AArch64ISD::SDIV_PRED, llvm::AArch64ISD::SETCC_MERGE_ZERO, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUO, llvm::AArch64ISD::SHL_PRED, llvm::AArch64ISD::SMAX_PRED, llvm::AArch64ISD::SMAXV, llvm::AArch64ISD::SMAXV_PRED, llvm::AArch64ISD::SMIN_PRED, llvm::AArch64ISD::SMINV, llvm::AArch64ISD::SMINV_PRED, llvm::AArch64ISD::SMULL, llvm::ISD::SPLAT_VECTOR, llvm::AArch64ISD::SRA_PRED, llvm::AArch64ISD::SRAD_MERGE_OP1, llvm::AArch64ISD::SRL_PRED, llvm::ISD::SSUBSAT, llvm::ISD::SUB, tryCombineCRC32(), tryCombineFixedPointConvert(), tryCombineLongOpWithDup(), tryCombineShiftImm(), tryConvertSVEWideCompare(), llvm::ISD::UADDSAT, llvm::AArch64ISD::UADDV, llvm::AArch64ISD::UADDV_PRED, llvm::AArch64ISD::UDIV_PRED, llvm::AArch64ISD::UMAX_PRED, llvm::AArch64ISD::UMAXV, llvm::AArch64ISD::UMAXV_PRED, llvm::AArch64ISD::UMIN_PRED, llvm::AArch64ISD::UMINV, llvm::AArch64ISD::UMINV_PRED, llvm::AArch64ISD::UMULL, llvm::ISD::USUBSAT, llvm::ISD::VSELECT, and llvm::ISD::XOR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 17799 of file AArch64ISelLowering.cpp.
References llvm::MemSDNode::getAlign(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SDValue::hasOneUse(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::ISD::isNormalLoad(), N, performVectorCompareAndMaskUnaryOpCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SINT_TO_FP, llvm::AArch64ISD::SITOF, and llvm::AArch64ISD::UITOF.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18528 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), getPTest(), getPTrue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), Idx, llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isScalableVector(), llvm::AArch64CC::LAST_ACTIVE, N, and llvm::ISD::VSCALE.
Referenced by performExtractVectorEltCombine().
|
static |
Definition at line 20982 of file AArch64ISelLowering.cpp.
References DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isInteger(), N, llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21034 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::isFloatingPoint(), llvm::AArch64ISD::LD1RO_MERGE_ZERO, llvm::AArch64ISD::LD1RQ_MERGE_ZERO, and N.
|
static |
Definition at line 21008 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isFloatingPoint(), N, llvm::ISD::NON_EXTLOAD, and llvm::ISD::UNINDEXED.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21908 of file AArch64ISelLowering.cpp.
References combineV3I8LoadExt(), llvm::commonAlignment(), llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::TypeSize::getFixed(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), I, llvm::ISD::INSERT_SUBVECTOR, llvm::AArch64Subtarget::isLittleEndian(), llvm::EVT::isScalableVector(), N, performTBISimplification(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::AArch64Subtarget::supportsAddressTopByteIgnored(), and llvm::ISD::TokenFactor.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 22417 of file AArch64ISelLowering.cpp.
References assert(), llvm::Data, DL, findMoreOptimalIndexType(), llvm::MaskedGatherScatterSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedGatherSDNode::getExtensionType(), llvm::MaskedGatherScatterSDNode::getIndex(), llvm::MaskedGatherScatterSDNode::getIndexType(), llvm::MaskedGatherScatterSDNode::getMask(), llvm::SelectionDAG::getMaskedGather(), llvm::SelectionDAG::getMaskedScatter(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::MaskedGatherSDNode::getPassThru(), llvm::MaskedGatherScatterSDNode::getScale(), llvm::MaskedScatterSDNode::getValue(), llvm::SelectionDAG::getVTList(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::MaskedScatterSDNode::isTruncatingStore(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 22239 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeVectorElementType(), DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::MaskedStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::getNumElementsFromSVEPredPattern(), llvm::MaskedStoreSDNode::getOffset(), getPTrue(), llvm::EVT::getSizeInBits(), llvm::MaskedStoreSDNode::getValue(), llvm::EVT::getVectorElementType(), llvm::Value::hasOneUse(), isHalvingTruncateOfLegalScalableType(), llvm::MaskedStoreSDNode::isTruncatingStore(), llvm::MaskedLoadStoreSDNode::isUnindexed(), N, llvm::AArch64ISD::PTRUE, trySimplifySrlAddToRshrnb(), llvm::AArch64ISD::UZP1, and llvm::EVT::widenIntegerVectorElementType().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 17516 of file AArch64ISelLowering.cpp.
References llvm::Add, llvm::ISD::ADD, llvm::APInt::ashr(), llvm::BitWidth, llvm::CallingConv::C, llvm::APInt::countr_zero(), DL, llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::APInt::getZExtValue(), llvm::SDNode::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::APInt::isNegative(), llvm::APInt::isNonNegative(), llvm::APInt::isPowerOf2(), isSignExtended(), IsSVECntIntrinsic(), isZeroExtended(), llvm::APInt::logBase2(), llvm::ISD::MUL, N, performMulVectorCmpZeroCombine(), performMulVectorExtendCombine(), RHS, llvm::APInt::sdivrem(), llvm::APInt::sge(), llvm::ISD::SHL, llvm::APInt::sle(), llvm::ISD::SUB, std::swap(), llvm::ISD::TRUNCATE, and X.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24484 of file AArch64ISelLowering.cpp.
References N, llvm::Intrinsic::not_intrinsic, tryCombineLongOpWithDup(), and tryCombineMULLWithUZP1().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 17483 of file AArch64ISelLowering.cpp.
References llvm::And, llvm::ISD::AND, llvm::AArch64ISD::CMLTz, DL, llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::ISD::isConstantSplatVector(), llvm::APInt::isMask(), N, llvm::AArch64ISD::NVCAST, and llvm::ISD::SRL.
Referenced by performMulCombine().
|
static |
Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
Definition at line 17463 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), llvm::Mul, and performBuildShuffleExtendCombine().
Referenced by performMulCombine().
|
static |
Definition at line 19326 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::CSEL, DL, getNegatedInteger(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), isNegatedInteger(), and N.
Referenced by performAddSubCombine().
|
static |
Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.
Definition at line 22455 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, Addr, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm::AArch64ISD::LD1x2post, llvm::AArch64ISD::LD1x3post, llvm::AArch64ISD::LD1x4post, llvm::AArch64ISD::LD2DUPpost, llvm::AArch64ISD::LD2LANEpost, llvm::AArch64ISD::LD2post, llvm::AArch64ISD::LD3DUPpost, llvm::AArch64ISD::LD3LANEpost, llvm::AArch64ISD::LD3post, llvm::AArch64ISD::LD4DUPpost, llvm::AArch64ISD::LD4LANEpost, llvm::AArch64ISD::LD4post, llvm_unreachable, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::AArch64ISD::ST1x2post, llvm::AArch64ISD::ST1x3post, llvm::AArch64ISD::ST1x4post, llvm::AArch64ISD::ST2LANEpost, llvm::AArch64ISD::ST2post, llvm::AArch64ISD::ST3LANEpost, llvm::AArch64ISD::ST3post, llvm::AArch64ISD::ST4LANEpost, and llvm::AArch64ISD::ST4post.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Get rid of unnecessary NVCASTs (that don't change the type).
Definition at line 23650 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), N, and llvm::AArch64ISD::NVCAST.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18155 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::isTypeLegal(), N, performANDORCSELCombine(), and tryCombineToBSL().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().
|
static |
Definition at line 9695 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, Cond, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::Value::hasOneUse(), I, llvm::isNullConstant(), isOrXorChain(), LHS, N, llvm::ISD::OR, RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::SmallVectorBase< Size_T >::size().
Referenced by performSETCCCombine().
|
static |
Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.
Definition at line 21697 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, Addr, llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::FMA, llvm::ISD::FMUL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::AArch64ISD::LD1DUPpost, llvm::AArch64ISD::LD1LANEpost, llvm::ISD::LOAD, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::Vector.
Referenced by performDUPCombine(), and performInsertVectorEltCombine().
Definition at line 18200 of file AArch64ISelLowering.cpp.
References llvm::SDValue::getValueType(), N, and llvm::AArch64ISD::REINTERPRET_CAST.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 24498 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), N, llvm::AArch64ISD::NVCAST, llvm::AArch64ISD::UADDLV, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23772 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, llvm::ISD::BITCAST, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isFloatingPoint(), llvm::MVT::isFloatingPoint(), llvm::EVT::isScalableVector(), isValidImmForSVEVecImmAddrMode(), N, llvm::Offset, llvm::AArch64ISD::SST1_IMM_PRED, llvm::AArch64ISD::SST1_PRED, llvm::AArch64ISD::SST1_UXTW_PRED, llvm::AArch64ISD::SST1Q_INDEX_PRED, llvm::AArch64ISD::SST1Q_PRED, llvm::AArch64ISD::SSTNT1_INDEX_PRED, llvm::AArch64ISD::SSTNT1_PRED, llvm::AArch64::SVEBitsPerBlock, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar.
This replaces any scalar setcc in the above pattern with a vector one followed by a DUP shuffle on the result.
Definition at line 23543 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::EVT::changeVectorElementTypeToInteger(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFloatingPoint(), llvm::EVT::isScalableVT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LHS, N, RHS, llvm::ISD::SCALAR_TO_VECTOR, and llvm::ISD::SETCC.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19127 of file AArch64ISelLowering.cpp.
References SetCCInfo::AArch64, llvm::ISD::ADD, assert(), GenericSetCCInfo::CC, AArch64SetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::AArch64ISD::CSEL, SetCCInfo::Generic, getAArch64Cmp(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), SetCCInfoAndKind::Info, SetCCInfoAndKind::IsAArch64, isSetCCOrZExtSetCC(), LHS, GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, RHS, and std::swap().
Referenced by performAddSubLongCombine().
|
static |
Definition at line 23146 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, Cond, llvm::AArch64ISD::CSEL, DL, llvm::SelectionDAG::getConstant(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::Value::hasOneUse(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVector(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::EVT::isScalarInteger(), LHS, N, performOrXorChainCombine(), RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, tryToWidenSetCCOperands(), llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_OR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().
|
static |
Definition at line 23284 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), Cond, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getNode(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), isAllActivePredicate(), isZerosVector(), LHS, N, performSetCCPunpkCombine(), RHS, llvm::AArch64ISD::SETCC_MERGE_ZERO, llvm::ISD::SETNE, and llvm::ISD::SIGN_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23245 of file AArch64ISelLowering.cpp.
References Cond, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), isZerosVector(), LHS, N, llvm::AArch64ISD::PTRUE, RHS, llvm::AArch64ISD::SETCC_MERGE_ZERO, llvm::ISD::SETNE, and llvm::ISD::SIGN_EXTEND.
Referenced by performSetccMergeZeroCombine().
|
static |
Definition at line 23992 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, EnableCombineMGatherIntrinsics, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1S_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_MERGE_ZERO, llvm::AArch64ISD::GLDNT1S_MERGE_ZERO, I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::AArch64ISD::LD1_MERGE_ZERO, llvm::AArch64ISD::LD1S_MERGE_ZERO, llvm::AArch64ISD::LDFF1_MERGE_ZERO, llvm::AArch64ISD::LDFF1S_MERGE_ZERO, llvm::AArch64ISD::LDNF1_MERGE_ZERO, llvm::AArch64ISD::LDNF1S_MERGE_ZERO, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SIGN_EXTEND_INREG, llvm::AArch64ISD::SUNPKHI, llvm::AArch64ISD::SUNPKLO, llvm::AArch64ISD::UUNPKHI, and llvm::AArch64ISD::UUNPKLO.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 20846 of file AArch64ISelLowering.cpp.
References assert(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::SDNode::getValueType(), isCheapToExtend(), llvm::EVT::isInteger(), N, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by performExtendCombine().
|
static |
Definition at line 21304 of file AArch64ISelLowering.cpp.
References assert(), N, and llvm::AArch64ISD::SPLICE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21055 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::Data, DL, llvm::SelectionDAG::getNode(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::EVT::isFloatingPoint(), N, and llvm::AArch64ISD::ST1_PRED.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21081 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::SelectionDAG::getMaskedStore(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::isFloatingPoint(), N, and llvm::ISD::UNINDEXED.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 22181 of file AArch64ISelLowering.cpp.
References combineBoolVectorAndTruncateStore(), combineI8TruncStore(), foldTruncStoreOfExt(), llvm::ISD::FP_ROUND, llvm::EVT::getFixedSizeInBits(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SelectionDAG::getTruncStore(), llvm::EVT::getVectorElementType(), llvm::Value::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFixedLengthVector(), isHalvingTruncateOfLegalScalableType(), N, performTBISimplification(), Ptr, splitStores(), llvm::AArch64Subtarget::supportsAddressTopByteIgnored(), trySimplifySrlAddToRshrnb(), and llvm::AArch64Subtarget::useSVEForFixedLengthVectors().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19750 of file AArch64ISelLowering.cpp.
References llvm::Add, llvm::ISD::ADD, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::M1(), llvm::ISD::MUL, N, llvm::peekThroughBitcasts(), llvm::AArch64ISD::SMULL, llvm::ISD::SUB, llvm::AArch64ISD::UMULL, and X.
Referenced by performAddSubCombine().
|
static |
Definition at line 22774 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ANDS, assert(), CC, DL, llvm::AArch64CC::EQ, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::getVTList(), llvm::AArch64CC::HI, llvm::APInt::isMask(), llvm::APInt::isPowerOf2(), llvm::AArch64CC::LO, N, and llvm::AArch64CC::NE.
Referenced by performCONDCombine().
|
static |
Definition at line 21678 of file AArch64ISelLowering.cpp.
References CC, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVectorIdxConstant(), N, and llvm::ISD::SIGN_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 18211 of file AArch64ISelLowering.cpp.
References llvm::And, llvm::ISD::AND, llvm::CallingConv::C, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, EnableCombineMGatherIntrinsics, llvm::ISD::EXTLOAD, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::AArch64ISD::GLD1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLD1_MERGE_ZERO, llvm::AArch64ISD::GLD1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_IMM_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_MERGE_ZERO, llvm::AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO, llvm::AArch64ISD::GLDNT1_MERGE_ZERO, isAllActivePredicate(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isConstantSplatVectorMaskForType(), llvm::AArch64ISD::LD1_MERGE_ZERO, llvm::AArch64ISD::LDFF1_MERGE_ZERO, llvm::AArch64ISD::LDNF1_MERGE_ZERO, N, llvm::ISD::SPLAT_VECTOR, llvm::AArch64ISD::UUNPKHI, llvm::AArch64ISD::UUNPKLO, and llvm::ISD::ZEXTLOAD.
Referenced by performANDCombine().
|
static |
Definition at line 19786 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, convertFromScalableVector(), convertToScalableVector(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isScalableVector(), llvm::AArch64ISD::MUL_PRED, N, and llvm::ISD::SUB.
Referenced by performAddSubCombine().
|
static |
Definition at line 24235 of file AArch64ISelLowering.cpp.
References llvm::EVT::changeVectorElementTypeToInteger(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), Idx, llvm::EVT::isInteger(), LHS, N, RHS, and llvm::ISD::VECTOR_SPLICE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Simplify Addr
given that the top byte of it is ignored by HW during address translation.
Definition at line 21813 of file AArch64ISelLowering.cpp.
References Addr, llvm::TargetLowering::DAGCombinerInfo::CommitTargetLoweringOpt(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by performLOADCombine(), and performSTORECombine().
|
static |
Definition at line 23403 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getTestBitOperand(), N, llvm::AArch64ISD::TBNZ, and llvm::AArch64ISD::TBZ.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19639 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::DUP, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::is64BitVector(), llvm::EVT::isFixedLengthVector(), N, and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 17150 of file AArch64ISelLowering.cpp.
References A, llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), performUADDVAddCombine(), llvm::AArch64ISD::SADDLP, llvm::ISD::SIGN_EXTEND, llvm::AArch64ISD::UADDLP, and llvm::ISD::ZERO_EXTEND.
Referenced by performUADDVAddCombine(), and performUADDVCombine().
|
static |
Definition at line 17237 of file AArch64ISelLowering.cpp.
References A, llvm::ISD::ADD, llvm::SelectionDAG::getNode(), N, performUADDVAddCombine(), and performUADDVZextCombine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 17199 of file AArch64ISelLowering.cpp.
References A, llvm::ISD::ADD, assert(), Concat, llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SDValue::getValueType(), llvm_unreachable, llvm::AArch64ISD::NVCAST, llvm::MVT::SimpleTy, llvm::AArch64ISD::UADDLV, and llvm::ISD::ZERO_EXTEND.
Referenced by performUADDVCombine().
|
static |
Definition at line 21314 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::changeVectorElementType(), DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedLoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::MaskedLoadSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SDValue::getNode(), llvm::getNumElementsFromSVEPredPattern(), llvm::MaskedLoadSDNode::getOffset(), llvm::MaskedLoadSDNode::getPassThru(), getPTrue(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::EVT::getVectorElementType(), llvm::SDNode::isUndef(), llvm::MaskedLoadStoreSDNode::isUnindexed(), isZerosVector(), llvm::ISD::MLOAD, N, llvm::AArch64ISD::PTRUE, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SEXTLOAD, llvm::AArch64ISD::UUNPKHI, llvm::AArch64ISD::UUNPKLO, and llvm::ISD::ZEXTLOAD.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 21444 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), isHalvingTruncateAndConcatOfLegalIntScalableType(), llvm::DataLayout::isLittleEndian(), llvm::EVT::isSimple(), llvm_unreachable, N, llvm::peekThroughBitcasts(), llvm::MVT::SimpleTy, llvm::ISD::TRUNCATE, tryCombineExtendRShTrunc(), trySimplifySrlAddToRshrnb(), llvm::ISD::UNDEF, llvm::AArch64ISD::UUNPKHI, llvm::AArch64ISD::UUNPKLO, llvm::AArch64ISD::UZP1, and X.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 17047 of file AArch64ISelLowering.cpp.
References A, llvm::ISD::ADD, B, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), I, llvm::EVT::isScalableVT(), llvm::ISD::MUL, N, performVecReduceAddCombineWithUADDLP(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::AArch64ISD::SDOT, llvm::ISD::SIGN_EXTEND, llvm::AArch64ISD::UDOT, llvm::ISD::VECREDUCE_ADD, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 16966 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ABS, llvm::ISD::ADD, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), N, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::AArch64ISD::UADDLP, llvm::ISD::VECREDUCE_ADD, and llvm::ISD::ZERO_EXTEND.
Referenced by performVecReduceAddCombine().
|
static |
Definition at line 23131 of file AArch64ISelLowering.cpp.
References DL, llvm::SDValue::getValueType(), getVectorBitwiseReduce(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isPow2VectorType(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 19473 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getContext(), llvm::EVT::getHalfSizedIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::EVT::isFixedLengthVector(), N, S1, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by performAddSubCombine().
|
static |
Definition at line 17753 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::isVector(), N, and llvm::ISD::SETCC.
Referenced by performIntToFpCombine().
|
static |
Optimize a vector shift instruction and its operand if shifted out bits are not used.
Definition at line 21650 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::APInt::getLowBitsSet(), N, llvm::TargetLowering::SimplifyDemandedBits(), llvm::AArch64ISD::VASHR, llvm::AArch64ISD::VLSHR, and llvm::AArch64ISD::VSHL.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 23472 of file AArch64ISelLowering.cpp.
References llvm::EVT::changeVectorElementTypeToInteger(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::ElementCount::getFixed(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::is_contained(), isAllActivePredicate(), isAllInactivePredicate(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::EVT::isFloatingPoint(), llvm::APInt::isOne(), llvm::EVT::isSimple(), N, llvm::Or, llvm::ISD::OR, llvm::ISD::SETCC, llvm::ISD::SETGT, llvm::MVT::SimpleTy, llvm::ISD::SRA, trySwapVSelectOperands(), and llvm::ISD::VSELECT.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
|
static |
Definition at line 17248 of file AArch64ISelLowering.cpp.
References foldVectorXorShiftIntoCmp(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
SDValue ReconstructShuffleWithRuntimeMask | ( | SDValue | Op, |
SelectionDAG & | DAG | ||
) |
Definition at line 11298 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BUILD_VECTOR, llvm::ISD::CONCAT_VECTORS, llvm::SmallVectorBase< Size_T >::empty(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isScalableVector(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
|
static |
Definition at line 11720 of file AArch64ISelLowering.cpp.
References assert(), llvm::sampleprof::Base, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::TRUNCATE, X, and Y.
Definition at line 24191 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isConstantSplatVectorAllZeros(), isLanes1toNKnownZero(), llvm::isNullConstant(), and N.
Referenced by performInsertVectorEltCombine().
|
static |
Definition at line 25145 of file AArch64ISelLowering.cpp.
References llvm::AArch64ISD::ADDP, assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::EVT::is256BitVector(), llvm::EVT::isFloatingPoint(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), Results, llvm::SelectionDAG::SplitVector(), and X.
|
static |
Definition at line 25419 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::BUILD_PAIR, getAtomicLoad128Opcode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::DataLayout::isBigEndian(), llvm::Lo, N, Ptr, Results, llvm::SelectionDAG::setNodeMemRefs(), llvm::SelectionDAG::SplitScalar(), std::swap(), and llvm::ISD::XOR.
|
static |
Definition at line 25050 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::DWARFExpression::Operation::getNumOperands(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), I, llvm::EVT::isVector(), N, Results, and vectorToScalarBitmask().
|
static |
Definition at line 25256 of file AArch64ISelLowering.cpp.
References llvm::Acquire, llvm::AcquireRelease, assert(), llvm::ISD::BUILD_PAIR, createGPRPairNode(), DL, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::DataLayout::isBigEndian(), llvm_unreachable, llvm::Lo, llvm::Monotonic, N, llvm::Release, Results, llvm::SequentiallyConsistent, llvm::SelectionDAG::setNodeMemRefs(), llvm::SelectionDAG::SplitScalar(), and std::swap().
|
static |
Definition at line 25193 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::Hi, llvm::Lo, N, Results, and llvm::SelectionDAG::SplitVectorOperand().
|
static |
Replace a splat of a scalar to a vector store by scalar stores of the scalar value.
The load store optimizer pass will merge them to store pair stores. This has better performance than a splat of the scalar followed by a split vector store. Even if the stores are not merged it is four stores vs a dup, followed by an ext.b and two stores.
Definition at line 21180 of file AArch64ISelLowering.cpp.
References llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), I, llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::isFloatingPoint(), llvm::StoreSDNode::isTruncatingStore(), and splitStoreSplat().
Referenced by splitStores().
|
static |
Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.
The load store optimizer pass will merge them to store pair stores. This should be better than a movi to create the vector zero followed by a vector store if the zero constant is not re-used, since one instructions and one register live range will be removed.
For example, the final generated code should be:
stp xzr, xzr, [x0]
instead of:
movi v0.2d, #0 str q0, [x0]
Definition at line 21113 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, DL, llvm::StoreSDNode::getBasePtr(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), I, llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::isNullConstant(), llvm::isNullFPConstant(), llvm::EVT::isScalableVector(), llvm::StoreSDNode::isTruncatingStore(), llvm::Offset, and splitStoreSplat().
Referenced by splitStores().
|
static |
Definition at line 12735 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::BuildVectorSDNode::isConstantSplat(), and llvm::APInt::zextOrTrunc().
Referenced by ConstantBuildVector(), and performANDCombine().
|
static |
Definition at line 4938 of file AArch64ISelLowering.cpp.
References DL, llvm::APInt::getHighBitsSet(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::SDValue::getValueType(), isAddSubSExt(), isAddSubZExt(), isExtendedBUILD_VECTOR(), isSignExtended(), isZeroExtended(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::SelectionDAG::SignBitIsZero(), llvm::AArch64ISD::SMULL, std::swap(), and llvm::AArch64ISD::UMULL.
|
static |
Set the IntrinsicInfo for the aarch64_sve_st<N>
intrinsics.
Definition at line 14864 of file AArch64ISelLowering.cpp.
References llvm::CallBase::arg_size(), assert(), DL, llvm::CallBase::getArgOperand(), llvm::Type::getContext(), llvm::TargetLoweringBase::getMemValueType(), llvm::EVT::getScalarType(), llvm::Value::getType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), I, Info, llvm::ISD::INTRINSIC_VOID, and llvm::MachineMemOperand::MOStore.
|
static |
Definition at line 15342 of file AArch64ISelLowering.cpp.
References llvm::sampleprof::Base, GEP, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by llvm::AArch64TargetLowering::shouldSinkOperands().
|
static |
We want to sink following cases: (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A, vscale.
Definition at line 15369 of file AArch64ISelLowering.cpp.
References llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_Mul(), llvm::PatternMatch::m_Shl(), llvm::PatternMatch::m_VScale(), llvm::PatternMatch::match(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by llvm::AArch64TargetLowering::shouldSinkOperands().
|
static |
Definition at line 4748 of file AArch64ISelLowering.cpp.
References addRequiredExtensionForVectorMULL(), assert(), llvm::ISD::BUILD_VECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::APInt::getHighBitsSet(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::EVT::is128BitVector(), llvm::ISD::isExtOpcode(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::TRUNCATE, and llvm::APInt::zextOrTrunc().
|
static |
Definition at line 21233 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::MemSDNode::getAlign(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::Function::hasMinSize(), llvm::EVT::isFixedLengthVector(), llvm::LSBaseSDNode::isIndexed(), llvm::MemSDNode::isVolatile(), N, replaceSplatVectorStore(), and replaceZeroVectorStore().
Referenced by performSTORECombine().
|
static |
Definition at line 20912 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::commonAlignment(), DL, llvm::MemSDNode::getAlign(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::MachineMemOperand::getFlags(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MachinePointerInfo::getWithOffset(), llvm::StoreSDNode::isTruncatingStore(), and llvm::Offset.
Referenced by replaceSplatVectorStore(), and replaceZeroVectorStore().
STATISTIC | ( | NumOptimizedImms | , |
"Number of times immediates were optimized" | |||
) |
STATISTIC | ( | NumShiftInserts | , |
"Number of vector shift inserts" | |||
) |
STATISTIC | ( | NumTailCalls | , |
"Number of tail calls" | |||
) |
|
static |
Definition at line 12832 of file AArch64ISelLowering.cpp.
References llvm::AArch64_AM::encodeAdvSIMDModImmType5(), llvm::AArch64_AM::encodeAdvSIMDModImmType6(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64_AM::isAdvSIMDModImmType5(), llvm::AArch64_AM::isAdvSIMDModImmType6(), llvm::EVT::isFixedLengthVector(), llvm::AArch64Subtarget::isNeonAvailable(), LHS, and llvm::AArch64ISD::NVCAST.
Referenced by ConstantBuildVector(), and performANDCombine().
|
static |
Definition at line 12779 of file AArch64ISelLowering.cpp.
References llvm::AArch64_AM::encodeAdvSIMDModImmType1(), llvm::AArch64_AM::encodeAdvSIMDModImmType2(), llvm::AArch64_AM::encodeAdvSIMDModImmType3(), llvm::AArch64_AM::encodeAdvSIMDModImmType4(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64_AM::isAdvSIMDModImmType1(), llvm::AArch64_AM::isAdvSIMDModImmType2(), llvm::AArch64_AM::isAdvSIMDModImmType3(), llvm::AArch64_AM::isAdvSIMDModImmType4(), llvm::EVT::isFixedLengthVector(), llvm::AArch64Subtarget::isNeonAvailable(), LHS, and llvm::AArch64ISD::NVCAST.
Referenced by ConstantBuildVector(), and performANDCombine().
|
static |
Definition at line 12877 of file AArch64ISelLowering.cpp.
References llvm::AArch64_AM::encodeAdvSIMDModImmType7(), llvm::AArch64_AM::encodeAdvSIMDModImmType8(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType7(), llvm::AArch64_AM::isAdvSIMDModImmType8(), and llvm::AArch64ISD::NVCAST.
Referenced by ConstantBuildVector().
|
static |
Definition at line 12758 of file AArch64ISelLowering.cpp.
References llvm::AArch64_AM::encodeAdvSIMDModImmType10(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType10(), and llvm::AArch64ISD::NVCAST.
Referenced by ConstantBuildVector().
|
static |
Definition at line 12908 of file AArch64ISelLowering.cpp.
References llvm::AArch64_AM::encodeAdvSIMDModImmType9(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType9(), and llvm::AArch64ISD::NVCAST.
Referenced by ConstantBuildVector().
|
static |
Definition at line 12929 of file AArch64ISelLowering.cpp.
References llvm::AArch64_AM::encodeAdvSIMDModImmType11(), llvm::AArch64_AM::encodeAdvSIMDModImmType12(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType11(), llvm::AArch64_AM::isAdvSIMDModImmType12(), and llvm::AArch64ISD::NVCAST.
Referenced by ConstantBuildVector().
|
static |
Definition at line 20274 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::INTRINSIC_WO_CHAIN, and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 21377 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPredicateForVector(), llvm::Hi, llvm::Lo, N, llvm::AArch64ISD::RSHRNB_I, llvm::AArch64ISD::URSHR_I_PRED, llvm::AArch64ISD::UUNPKHI, llvm::AArch64ISD::UUNPKLO, and llvm::AArch64ISD::UZP1.
Referenced by performUzpCombine().
|
static |
Definition at line 18923 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and N.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 20147 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, llvm::Intrinsic::not_intrinsic, RHS, and tryExtendDUPToExtractHigh().
Referenced by performExtendCombine(), performIntrinsicCombine(), and performMULLCombine().
|
static |
Definition at line 24351 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, DL, llvm::AArch64ISD::DUP, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::User::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), llvm::isNullConstant(), llvm::SelectionDAG::isSplatValue(), LHS, N, llvm::SelectionDAG::ReplaceAllUsesWith(), RHS, llvm::SDValue::setNode(), llvm::ISD::TRUNCATE, llvm::SDNode::use_begin(), llvm::SDNode::use_size(), llvm::SDNode::uses(), and llvm::AArch64ISD::UZP1.
Referenced by performMULLCombine().
|
static |
Definition at line 20180 of file AArch64ISelLowering.cpp.
References llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::APInt::getSExtValue(), llvm::MVT::getSizeInBits(), llvm::BuildVectorSDNode::isConstantSplat(), llvm_unreachable, N, llvm::ISD::SCALAR_TO_VECTOR, llvm::AArch64ISD::SQSHL_I, llvm::AArch64ISD::SQSHLU_I, llvm::AArch64ISD::SRSHR_I, llvm::AArch64ISD::UQSHL_I, llvm::AArch64ISD::URSHR_I, llvm::AArch64ISD::VASHR, llvm::AArch64ISD::VLSHR, and llvm::AArch64ISD::VSHL.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 17979 of file AArch64ISelLowering.cpp.
References llvm::Add, llvm::ISD::ADD, llvm::ISD::AND, llvm::AArch64ISD::BSP, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::EVT::isScalableVector(), llvm::EVT::isVector(), N, llvm::ISD::SUB, and llvm::AArch64TargetLowering::useSVEForFixedLengthVectorVT().
Referenced by performORCombine().
|
static |
Definition at line 20353 of file AArch64ISelLowering.cpp.
References CC, DL, llvm::AArch64ISD::DUP, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), getIntrinsicID(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm_unreachable, N, llvm::AArch64ISD::SETCC_MERGE_ZERO, llvm::Splat, and llvm::ISD::SPLAT_VECTOR.
Referenced by performIntrinsicCombine().
|
static |
Definition at line 18986 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64ISD::DUP, llvm::AArch64ISD::DUPLANE16, llvm::AArch64ISD::DUPLANE32, llvm::AArch64ISD::DUPLANE64, llvm::AArch64ISD::DUPLANE8, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is64BitVector(), llvm::AArch64ISD::MOVI, llvm::AArch64ISD::MOVIedit, llvm::AArch64ISD::MOVImsl, llvm::AArch64ISD::MOVIshift, llvm::AArch64ISD::MVNImsl, llvm::AArch64ISD::MVNIshift, and N.
Referenced by performAddSubLongCombine(), and tryCombineLongOpWithDup().
|
static |
Definition at line 11996 of file AArch64ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), and isConcatMask().
Definition at line 21994 of file AArch64ISelLowering.cpp.
References assert(), llvm::Depth, llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::INVALID_SIMPLE_VALUE_TYPE, llvm::EVT::isSimple(), llvm::EVT::isVector(), llvm::ISD::SETCC, llvm::ISD::TRUNCATE, and tryGetOriginalBoolVectorType().
Referenced by tryGetOriginalBoolVectorType(), and vectorToScalarBitmask().
|
static |
Definition at line 13032 of file AArch64ISelLowering.cpp.
References llvm::And, llvm::ISD::AND, assert(), llvm::AArch64ISD::BICi, llvm::CallingConv::C, llvm::dbgs(), DL, llvm::SDNode::dump(), llvm::ConstantSDNode::getAPIntValue(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), isAllActivePredicate(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), LLVM_DEBUG, N, llvm::AArch64ISD::SHL_PRED, llvm::AArch64ISD::SRL_PRED, llvm::AArch64ISD::VLSHR, llvm::AArch64ISD::VSHL, llvm::AArch64ISD::VSLI, llvm::AArch64ISD::VSRI, X, Y, and llvm::APInt::zextOrTrunc().
|
static |
Definition at line 21417 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, canLowerSRLToRoundingShiftForVT(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), llvm::EVT::isScalableVector(), and llvm::AArch64ISD::RSHRNB_I.
Referenced by performMSTORECombine(), performSTORECombine(), and performUzpCombine().
|
static |
Definition at line 23434 of file AArch64ISelLowering.cpp.
References CC, llvm::ISD::FADD, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), N, llvm::ISD::SETCC, and llvm::ISD::VSELECT.
Referenced by performVSelectCombine().
|
static |
Definition at line 12417 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::C, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), I, and llvm::ISD::INTRINSIC_WO_CHAIN.
|
static |
Definition at line 23085 of file AArch64ISelLowering.cpp.
References llvm::any_of(), CC, llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDNode::getOpcode(), llvm::EVT::getScalarSizeInBits(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), N, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::ISD::VSELECT, and llvm::ISD::ZERO_EXTEND.
Referenced by performSETCCCombine().
|
static |
Definition at line 12384 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getBitcast(), llvm::EVT::getFixedSizeInBits(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), and isWideTypeMask().
|
static |
Definition at line 26043 of file AArch64ISelLowering.cpp.
References llvm::IRBuilderBase::CreateCall(), llvm::IRBuilderBase::CreateConstGEP1_32(), llvm::IRBuilderBase::CreatePointerCast(), llvm::Intrinsic::getDeclaration(), llvm::IRBuilderBase::GetInsertBlock(), llvm::IRBuilderBase::getInt8Ty(), llvm::GlobalValue::getParent(), llvm::BasicBlock::getParent(), llvm::IRBuilderBase::getPtrTy(), and llvm::Offset.
Referenced by llvm::AArch64TargetLowering::getIRStackGuard(), and llvm::AArch64TargetLowering::getSafeStackPointerLocation().
|
static |
Definition at line 3903 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), and llvm::AArch64ISD::SUBS.
Referenced by lowerADDSUBO_CARRY().
|
static |
Definition at line 22026 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, llvm::EVT::changeVectorElementTypeToInteger(), DL, llvm::AArch64ISD::EXT, llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::EVT::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), tryGetOriginalBoolVectorType(), llvm::ISD::VECREDUCE_ADD, and llvm::AArch64ISD::ZIP1.
Referenced by combineBoolVectorAndTruncateStore(), and replaceBoolVectorBitcast().
|
static |
WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class.
Definition at line 11277 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, and WidenVector().
Referenced by constructDup(), GeneratePerfectShuffle(), performConcatVectorsCombine(), and WidenVector().
cl::opt< bool > EnableAArch64ELFLocalDynamicTLSGeneration("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false)) | ( | "aarch64-elf-ldtls-generation" | , |
cl::Hidden | , | ||
cl::desc("Allow AArch64 Local Dynamic TLS code generation") | , | ||
cl::init(false) | |||
) |
Referenced by llvm::AArch64MCInstLower::lowerSymbolOperandELF().
|
static |
Referenced by performSignExtendInRegCombine(), and performSVEAndCombine().
|
static |
Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant().
Definition at line 154 of file AArch64ISelLowering.cpp.
Referenced by llvm::AArch64::getFPRArgRegs().
Definition at line 151 of file AArch64ISelLowering.cpp.
Referenced by f64AssignAAPCS(), and llvm::AArch64::getGPRArgRegs().
|
static |
Referenced by isOrXorChain().
Value type used for condition codes.
Definition at line 149 of file AArch64ISelLowering.cpp.
Referenced by emitComparison(), emitConditionalComparison(), getAArch64Cmp(), getAArch64XALUOOp(), and performANDORCSELCombine().