LLVM 17.0.0git
|
Post-legalization lowering for instructions. More...
#include "AArch64GlobalISelUtils.h"
#include "AArch64Subtarget.h"
#include "AArch64TargetMachine.h"
#include "GISel/AArch64LegalizerInfo.h"
#include "MCTargetDesc/AArch64MCTargetDesc.h"
#include "TargetInfo/AArch64TargetInfo.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/CodeGen/GlobalISel/Combiner.h"
#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/InitializePasses.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include <optional>
#include "AArch64GenPostLegalizeGILowering.inc"
Go to the source code of this file.
Classes | |
struct | ShuffleVectorPseudo |
Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR. More... | |
Namespaces | |
namespace | llvm |
This is an optimization pass for GlobalISel generic memory operations. | |
Macros | |
#define | DEBUG_TYPE "aarch64-postlegalizer-lowering" |
#define | AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS |
#define | AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H |
#define | AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_CPP |
Functions | |
static bool | isREVMask (ArrayRef< int > M, unsigned EltSize, unsigned NumElts, unsigned BlockSize) |
Check if a vector shuffle corresponds to a REV instruction with the specified blocksize. | |
static bool | isTRNMask (ArrayRef< int > M, unsigned NumElts, unsigned &WhichResult) |
Determines if M is a shuffle vector mask for a TRN of NumElts . | |
static std::optional< std::pair< bool, uint64_t > > | getExtMask (ArrayRef< int > M, unsigned NumElts) |
Check if a G_EXT instruction can handle a shuffle mask M when the vector sources of the shuffle are different. | |
static bool | isUZPMask (ArrayRef< int > M, unsigned NumElts, unsigned &WhichResult) |
Determines if M is a shuffle vector mask for a UZP of NumElts . | |
static bool | isZipMask (ArrayRef< int > M, unsigned NumElts, unsigned &WhichResult) |
static std::optional< std::pair< bool, int > > | isINSMask (ArrayRef< int > M, int NumInputElements) |
Helper function for matchINS. | |
static bool | matchREV (MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo) |
static bool | matchTRN (MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo) |
static bool | matchUZP (MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo) |
static bool | matchZip (MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo) |
static bool | matchDupFromInsertVectorElt (int Lane, MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo) |
Helper function for matchDup. | |
static bool | matchDupFromBuildVector (int Lane, MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo) |
Helper function for matchDup. | |
static bool | matchDup (MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo) |
static bool | isSingletonExtMask (ArrayRef< int > M, LLT Ty) |
static bool | matchEXT (MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo) |
static bool | applyShuffleVectorPseudo (MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) |
Replace a G_SHUFFLE_VECTOR instruction with a pseudo. | |
static bool | applyEXT (MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) |
Replace a G_SHUFFLE_VECTOR instruction with G_EXT. | |
static bool | matchINS (MachineInstr &MI, MachineRegisterInfo &MRI, std::tuple< Register, int, Register, int > &MatchInfo) |
Match a G_SHUFFLE_VECTOR with a mask which corresponds to a G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_ELT pair. | |
static bool | applyINS (MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &Builder, std::tuple< Register, int, Register, int > &MatchInfo) |
static bool | isVShiftRImm (Register Reg, MachineRegisterInfo &MRI, LLT Ty, int64_t &Cnt) |
isVShiftRImm - Check if this is a valid vector for the immediate operand of a vector shift right operation. | |
static bool | matchVAshrLshrImm (MachineInstr &MI, MachineRegisterInfo &MRI, int64_t &Imm) |
Match a vector G_ASHR or G_LSHR with a valid immediate shift. | |
static bool | applyVAshrLshrImm (MachineInstr &MI, MachineRegisterInfo &MRI, int64_t &Imm) |
std::optional< std::pair< uint64_t, CmpInst::Predicate > > | tryAdjustICmpImmAndPred (Register RHS, CmpInst::Predicate P, const MachineRegisterInfo &MRI) |
Determine if it is possible to modify the RHS and predicate P of a G_ICMP instruction such that the right-hand side is an arithmetic immediate. | |
bool | matchAdjustICmpImmAndPred (MachineInstr &MI, const MachineRegisterInfo &MRI, std::pair< uint64_t, CmpInst::Predicate > &MatchInfo) |
Determine whether or not it is possible to update the RHS and predicate of a G_ICMP instruction such that the RHS will be selected as an arithmetic immediate. | |
bool | applyAdjustICmpImmAndPred (MachineInstr &MI, std::pair< uint64_t, CmpInst::Predicate > &MatchInfo, MachineIRBuilder &MIB, GISelChangeObserver &Observer) |
bool | matchDupLane (MachineInstr &MI, MachineRegisterInfo &MRI, std::pair< unsigned, int > &MatchInfo) |
bool | applyDupLane (MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, std::pair< unsigned, int > &MatchInfo) |
static bool | matchBuildVectorToDup (MachineInstr &MI, MachineRegisterInfo &MRI) |
static bool | applyBuildVectorToDup (MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B) |
static unsigned | getCmpOperandFoldingProfit (Register CmpOp, const MachineRegisterInfo &MRI) |
static bool | trySwapICmpOperands (MachineInstr &MI, const MachineRegisterInfo &MRI) |
static bool | applySwapICmpOperands (MachineInstr &MI, GISelChangeObserver &Observer) |
static std::function< Register(MachineIRBuilder &)> | getVectorFCMP (AArch64CC::CondCode CC, Register LHS, Register RHS, bool IsZero, bool NoNans, MachineRegisterInfo &MRI) |
static bool | lowerVectorFCMP (MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIB) |
Try to lower a vector G_FCMP MI into an AArch64-specific pseudo. | |
static bool | matchFormTruncstore (MachineInstr &MI, MachineRegisterInfo &MRI, Register &SrcReg) |
static bool | applyFormTruncstore (MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, GISelChangeObserver &Observer, Register &SrcReg) |
static bool | matchVectorSextInReg (MachineInstr &MI, MachineRegisterInfo &MRI) |
static void | applyVectorSextInReg (MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, GISelChangeObserver &Observer) |
INITIALIZE_PASS_BEGIN (AArch64PostLegalizerLowering, DEBUG_TYPE, "Lower AArch64 MachineInstrs after legalization", false, false) INITIALIZE_PASS_END(AArch64PostLegalizerLowering | |
FunctionPass * | llvm::createAArch64PostLegalizerLowering () |
Variables | |
DEBUG_TYPE | |
Lower AArch64 MachineInstrs after | legalization |
Lower AArch64 MachineInstrs after | false |
Post-legalization lowering for instructions.
This is used to offload pattern matching from the selector.
For example, this combiner will notice that a G_SHUFFLE_VECTOR is actually a G_ZIP, G_UZP, etc.
General optimization combines should be handled by either the AArch64PostLegalizerCombiner or the AArch64PreLegalizerCombiner.
Definition in file AArch64PostLegalizerLowering.cpp.
#define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_CPP |
Definition at line 1084 of file AArch64PostLegalizerLowering.cpp.
#define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS |
Definition at line 1051 of file AArch64PostLegalizerLowering.cpp.
#define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H |
Definition at line 1056 of file AArch64PostLegalizerLowering.cpp.
#define DEBUG_TYPE "aarch64-postlegalizer-lowering" |
Definition at line 49 of file AArch64PostLegalizerLowering.cpp.
bool applyAdjustICmpImmAndPred | ( | MachineInstr & | MI, |
std::pair< uint64_t, CmpInst::Predicate > & | MatchInfo, | ||
MachineIRBuilder & | MIB, | ||
GISelChangeObserver & | Observer | ||
) |
Definition at line 671 of file AArch64PostLegalizerLowering.cpp.
References llvm::MachineIRBuilder::buildConstant(), llvm::GISelChangeObserver::changedInstr(), llvm::GISelChangeObserver::changingInstr(), llvm::MachineIRBuilder::getMRI(), MI, MRI, RHS, and llvm::MachineIRBuilder::setInstrAndDebugLoc().
|
static |
Definition at line 778 of file AArch64PostLegalizerLowering.cpp.
bool applyDupLane | ( | MachineInstr & | MI, |
MachineRegisterInfo & | MRI, | ||
MachineIRBuilder & | B, | ||
std::pair< unsigned, int > & | MatchInfo | ||
) |
Definition at line 738 of file AArch64PostLegalizerLowering.cpp.
References assert(), B, llvm::LLT::changeElementCount(), llvm::LLT::fixed_vector(), llvm::ElementCount::getFixed(), MI, MRI, and llvm::LLT::scalar().
|
static |
Replace a G_SHUFFLE_VECTOR instruction with G_EXT.
Special-cased because the constant operand must be emitted as a G_CONSTANT for the imported tablegen patterns to work.
Definition at line 449 of file AArch64PostLegalizerLowering.cpp.
References llvm::MachineIRBuilder::buildConstant(), llvm::MachineIRBuilder::buildInstr(), MI, ShuffleVectorPseudo::Opc, llvm::LLT::scalar(), and ShuffleVectorPseudo::SrcOps.
|
static |
Definition at line 1021 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::GISelChangeObserver::changedInstr(), llvm::GISelChangeObserver::changingInstr(), and MI.
|
static |
Definition at line 498 of file AArch64PostLegalizerLowering.cpp.
References Builder, MI, MRI, and llvm::LLT::scalar().
|
static |
Replace a G_SHUFFLE_VECTOR instruction with a pseudo.
Opc
is the opcode to use. MI
is the G_SHUFFLE_VECTOR.
Definition at line 438 of file AArch64PostLegalizerLowering.cpp.
References llvm::MachineIRBuilder::buildInstr(), MI, ShuffleVectorPseudo::Opc, and ShuffleVectorPseudo::SrcOps.
|
static |
Definition at line 881 of file AArch64PostLegalizerLowering.cpp.
References llvm::GISelChangeObserver::changedInstr(), llvm::CmpInst::getSwappedPredicate(), LHS, MI, and RHS.
|
static |
Definition at line 541 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::MachineIRBuilder::buildConstant(), llvm::MachineIRBuilder::buildInstr(), MI, and llvm::LLT::scalar().
|
static |
Definition at line 1042 of file AArch64PostLegalizerLowering.cpp.
References assert(), B, llvm::LegalizerHelper::lower(), and MI.
|
static |
Definition at line 789 of file AArch64PostLegalizerLowering.cpp.
References llvm::getDefIgnoringCopies(), llvm::getIConstantVRegValWithLookThrough(), llvm::LLT::getSizeInBits(), llvm::LLT::isVector(), MI, and MRI.
Referenced by trySwapICmpOperands().
|
static |
Check if a G_EXT instruction can handle a shuffle mask M
when the vector sources of the shuffle are different.
Definition at line 115 of file AArch64PostLegalizerLowering.cpp.
References llvm::any_of(), llvm::find_if(), llvm::APInt::getZExtValue(), llvm::APInt::logBase2(), and llvm::make_range().
Referenced by matchEXT().
|
static |
CC
. [in] | IsZero | - True if the comparison is against 0. |
[in] | NoNans | - True if the target has NoNansFPMath. |
Definition at line 899 of file AArch64PostLegalizerLowering.cpp.
References assert(), CC, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, getReg(), llvm::AArch64CC::GT, llvm::LLT::isVector(), LHS, llvm_unreachable, llvm::AArch64CC::LS, llvm::AArch64CC::MI, MRI, llvm::AArch64CC::NE, and RHS.
Referenced by lowerVectorFCMP().
INITIALIZE_PASS_BEGIN | ( | AArch64PostLegalizerLowering | , |
DEBUG_TYPE | , | ||
"Lower AArch64 MachineInstrs after legalization" | , | ||
false | , | ||
false | |||
) |
|
static |
Helper function for matchINS.
M
is an ins mask for NumInputElements
.First element of the returned pair is true when the produced G_INSERT_VECTOR_ELT destination should be the LHS of the G_SHUFFLE_VECTOR.
Second element is the destination lane for the G_INSERT_VECTOR_ELT.
Definition at line 197 of file AArch64PostLegalizerLowering.cpp.
References Idx.
Referenced by matchINS().
|
static |
Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.
Definition at line 70 of file AArch64PostLegalizerLowering.cpp.
References assert(), and BlockSize.
Referenced by matchREV().
Definition at line 379 of file AArch64PostLegalizerLowering.cpp.
References llvm::LLT::getNumElements(), and I.
Referenced by matchEXT().
Determines if M
is a shuffle vector mask for a TRN of NumElts
.
Whether or not G_TRN1 or G_TRN2 should be used is stored in WhichResult
.
Definition at line 99 of file AArch64PostLegalizerLowering.cpp.
Referenced by matchTRN().
Determines if M
is a shuffle vector mask for a UZP of NumElts
.
Whether or not G_UZP1 or G_UZP2 should be used is stored in WhichResult
.
Definition at line 157 of file AArch64PostLegalizerLowering.cpp.
Referenced by matchUZP().
|
static |
isVShiftRImm - Check if this is a valid vector for the immediate operand of a vector shift right operation.
The value must be in the range: 1 <= Value <= ElementBits for a right shift.
Definition at line 518 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::AArch64GISelUtils::getAArch64VectorSplatScalar(), llvm::LLT::getScalarSizeInBits(), llvm::LLT::isVector(), MI, and MRI.
Referenced by matchVAshrLshrImm().
M
is a zip mask for a shuffle vector of NumElts
. Whether or not G_ZIP1 or G_ZIP2 should be used is stored in WhichResult
. Definition at line 172 of file AArch64PostLegalizerLowering.cpp.
References Idx.
Referenced by matchZip().
|
static |
Try to lower a vector G_FCMP MI
into an AArch64-specific pseudo.
Definition at line 953 of file AArch64PostLegalizerLowering.cpp.
References llvm::AArch64CC::AL, assert(), llvm::MachineIRBuilder::buildNot(), llvm::MachineIRBuilder::buildOr(), CC, llvm::AArch64GISelUtils::changeVectorFCMPPredToAArch64CC(), llvm::AArch64CC::EQ, llvm::AArch64GISelUtils::getAArch64VectorSplat(), llvm::MachineInstrBuilder::getReg(), getVectorFCMP(), llvm::LLT::isVector(), LHS, MI, MRI, RHS, and llvm::MachineIRBuilder::setInstrAndDebugLoc().
bool matchAdjustICmpImmAndPred | ( | MachineInstr & | MI, |
const MachineRegisterInfo & | MRI, | ||
std::pair< uint64_t, CmpInst::Predicate > & | MatchInfo | ||
) |
Determine whether or not it is possible to update the RHS and predicate of a G_ICMP instruction such that the RHS will be selected as an arithmetic immediate.
MI
- The G_ICMP instruction MatchInfo
- The new RHS immediate and predicate on success
See tryAdjustICmpImmAndPred for valid transformations.
Definition at line 658 of file AArch64PostLegalizerLowering.cpp.
References assert(), MI, MRI, RHS, and tryAdjustICmpImmAndPred().
|
static |
Definition at line 764 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::AArch64GISelUtils::getAArch64VectorSplat(), MI, and MRI.
|
static |
Definition at line 360 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::getSplatIndex(), matchDupFromBuildVector(), matchDupFromInsertVectorElt(), MI, and MRI.
|
static |
Helper function for matchDup.
Definition at line 344 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::getOpcodeDef(), MI, and MRI.
Referenced by matchDup().
|
static |
Helper function for matchDup.
Definition at line 305 of file AArch64PostLegalizerLowering.cpp.
References llvm::getOpcodeDef(), llvm::MIPatternMatch::m_ZeroInt(), MI, llvm::MIPatternMatch::mi_match(), and MRI.
Referenced by matchDup().
bool matchDupLane | ( | MachineInstr & | MI, |
MachineRegisterInfo & | MRI, | ||
std::pair< unsigned, int > & | MatchInfo | ||
) |
Definition at line 686 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::LLT::getElementType(), llvm::LLT::getNumElements(), llvm::LLT::getSizeInBits(), llvm::getSplatIndex(), MI, and MRI.
|
static |
Definition at line 406 of file AArch64PostLegalizerLowering.cpp.
References assert(), getExtMask(), llvm::LLT::getNumElements(), isSingletonExtMask(), MI, MRI, and std::swap().
|
static |
Definition at line 1008 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::MIPatternMatch::m_GTrunc(), llvm::MIPatternMatch::m_Reg(), MI, llvm::MIPatternMatch::mi_match(), and MRI.
|
static |
Match a G_SHUFFLE_VECTOR with a mask which corresponds to a G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_ELT pair.
e.g. shuf = G_SHUFFLE_VECTOR left, right, shufflemask(0, 0)
Can be represented as
extract = G_EXTRACT_VECTOR_ELT left, 0 ins = G_INSERT_VECTOR_ELT left, extract, 1
Definition at line 471 of file AArch64PostLegalizerLowering.cpp.
References assert(), isINSMask(), llvm::Left, MI, MRI, and llvm::Right.
|
static |
MI
can be replaced with a G_REV instruction. Returns the appropriate G_REV opcode in Opc
. Definition at line 222 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::LLT::getNumElements(), llvm::LLT::getScalarSizeInBits(), isREVMask(), MI, and MRI.
|
static |
MI
can be replaced with a G_TRN1 or G_TRN2 instruction. Definition at line 251 of file AArch64PostLegalizerLowering.cpp.
References assert(), isTRNMask(), MI, and MRI.
|
static |
MI
can be replaced with a G_UZP1 or G_UZP2 instruction.[in] | MI | - The shuffle vector instruction. |
[out] | MatchInfo | - Either G_UZP1 or G_UZP2 on success. |
Definition at line 272 of file AArch64PostLegalizerLowering.cpp.
References assert(), isUZPMask(), MI, and MRI.
|
static |
Match a vector G_ASHR or G_LSHR with a valid immediate shift.
Definition at line 531 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::LLT::isVector(), isVShiftRImm(), MI, and MRI.
|
static |
Definition at line 1035 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::LLT::isVector(), MI, and MRI.
|
static |
Definition at line 288 of file AArch64PostLegalizerLowering.cpp.
References assert(), isZipMask(), MI, and MRI.
std::optional< std::pair< uint64_t, CmpInst::Predicate > > tryAdjustICmpImmAndPred | ( | Register | RHS, |
CmpInst::Predicate | P, | ||
const MachineRegisterInfo & | MRI | ||
) |
Determine if it is possible to modify the RHS
and predicate P
of a G_ICMP instruction such that the right-hand side is an arithmetic immediate.
Definition at line 562 of file AArch64PostLegalizerLowering.cpp.
References assert(), llvm::CallingConv::C, llvm::getIConstantVRegValWithLookThrough(), llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_SLT, llvm::CmpInst::ICMP_UGE, llvm::CmpInst::ICMP_UGT, llvm::CmpInst::ICMP_ULE, llvm::CmpInst::ICMP_ULT, INT64_MAX, INT64_MIN, isLegalArithImmed(), MRI, P, RHS, Size, and UINT64_MAX.
Referenced by matchAdjustICmpImmAndPred().
|
static |
MI
. Definition at line 844 of file AArch64PostLegalizerLowering.cpp.
References assert(), getCmpOperandFoldingProfit(), llvm::getDefIgnoringCopies(), llvm::getIConstantVRegValWithLookThrough(), isCMN(), isLegalArithImmed(), LHS, MI, MRI, and RHS.
DEBUG_TYPE |
Definition at line 1134 of file AArch64PostLegalizerLowering.cpp.
Lower AArch64 MachineInstrs after false |
Definition at line 1135 of file AArch64PostLegalizerLowering.cpp.
Lower AArch64 MachineInstrs after legalization |
Definition at line 1135 of file AArch64PostLegalizerLowering.cpp.