LLVM 20.0.0git
Macros | Enumerations | Functions | Variables
AArch64InstrInfo.cpp File Reference
#include "AArch64InstrInfo.h"
#include "AArch64ExpandImm.h"
#include "AArch64FrameLowering.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64PointerAuth.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "MCTargetDesc/AArch64MCTargetDesc.h"
#include "Utils/AArch64BaseInfo.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineCombinerPattern.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/DebugInfoMetadata.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Module.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCInstBuilder.h"
#include "llvm/MC/MCInstrDesc.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CodeGen.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/LEB128.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <cassert>
#include <cstdint>
#include <iterator>
#include <utility>
#include "AArch64GenInstrInfo.inc"

Go to the source code of this file.

Macros

#define GET_INSTRINFO_CTOR_DTOR
 
#define GET_INSTRINFO_HELPERS
 
#define GET_INSTRMAP_INFO
 

Enumerations

enum  AccessKind { AK_Write = 0x01 , AK_Read = 0x10 , AK_All = 0x11 }
 
enum class  FMAInstKind { Default , Indexed , Accumulator }
 
enum  MachineOutlinerClass {
  MachineOutlinerDefault , MachineOutlinerTailCall , MachineOutlinerNoLRSave , MachineOutlinerThunk ,
  MachineOutlinerRegSave , MachineOutlinerTailCall , MachineOutlinerThunk , MachineOutlinerNoLRSave ,
  MachineOutlinerRegSave , MachineOutlinerDefault , MachineOutlinerDefault , MachineOutlinerTailCall
}
 Constants defining how certain sequences should be outlined. More...
 
enum  MachineOutlinerMBBFlags {
  LRUnavailableSomewhere = 0x2 , HasCalls = 0x4 , UnsafeRegsDead = 0x8 , LRUnavailableSomewhere = 0x2 ,
  HasCalls = 0x4 , UnsafeRegsDead = 0x8
}
 

Functions

static void parseCondBranch (MachineInstr *LastInst, MachineBasicBlock *&Target, SmallVectorImpl< MachineOperand > &Cond)
 
static unsigned getBranchDisplacementBits (unsigned Opc)
 
static unsigned removeCopies (const MachineRegisterInfo &MRI, unsigned VReg)
 
static unsigned canFoldIntoCSel (const MachineRegisterInfo &MRI, unsigned VReg, unsigned *NewVReg=nullptr)
 
static bool isCheapImmediate (const MachineInstr &MI, unsigned BitSize)
 
static bool UpdateOperandRegClass (MachineInstr &Instr)
 
static unsigned convertToNonFlagSettingOpc (const MachineInstr &MI)
 Return the opcode that does not set flags when possible - otherwise return the original opcode.
 
static bool areCFlagsAccessedBetweenInstrs (MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, const TargetRegisterInfo *TRI, const AccessKind AccessToCheck=AK_All)
 True when condition flags are accessed (either by writing or reading) on the instruction trace starting at From and ending at To.
 
static unsigned sForm (MachineInstr &Instr)
 Get opcode of S version of Instr.
 
static bool areCFlagsAliveInSuccessors (const MachineBasicBlock *MBB)
 Check if AArch64::NZCV should be alive in successors of MBB.
 
static int findCondCodeUseOperandIdxForBranchOrSelect (const MachineInstr &Instr)
 
static AArch64CC::CondCode findCondCodeUsedByInstr (const MachineInstr &Instr)
 Find a condition code used by the instruction.
 
static UsedNZCV getUsedNZCV (AArch64CC::CondCode CC)
 
static bool isADDSRegImm (unsigned Opcode)
 
static bool isSUBSRegImm (unsigned Opcode)
 
static bool canInstrSubstituteCmpInstr (MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI)
 Check if CmpInstr can be substituted by MI.
 
static bool canCmpInstrBeRemoved (MachineInstr &MI, MachineInstr &CmpInstr, int CmpValue, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > &CCUseInstrs, bool &IsInvertCC)
 
static unsigned regOffsetOpcode (unsigned Opcode)
 
unsigned scaledOffsetOpcode (unsigned Opcode, unsigned &Scale)
 
unsigned unscaledOffsetOpcode (unsigned Opcode)
 
static unsigned offsetExtendOpcode (unsigned Opcode)
 
static bool isPostIndexLdStOpcode (unsigned Opcode)
 Return true if the opcode is a post-index ld/st instruction, which really loads from base+0.
 
static const TargetRegisterClassgetRegClass (const MachineInstr &MI, Register Reg)
 
static bool scaleOffset (unsigned Opc, int64_t &Offset)
 
static bool canPairLdStOpc (unsigned FirstOpc, unsigned SecondOpc)
 
static bool shouldClusterFI (const MachineFrameInfo &MFI, int FI1, int64_t Offset1, unsigned Opcode1, int FI2, int64_t Offset2, unsigned Opcode2)
 
static const MachineInstrBuilderAddSubReg (const MachineInstrBuilder &MIB, unsigned Reg, unsigned SubIdx, unsigned State, const TargetRegisterInfo *TRI)
 
static bool forwardCopyWillClobberTuple (unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
 
static void storeRegPairToStackSlot (const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register SrcReg, bool IsKill, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
 
static void loadRegPairFromStackSlot (const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register DestReg, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
 
static void appendVGScaledOffsetExpr (SmallVectorImpl< char > &Expr, int NumBytes, int NumVGScaledBytes, unsigned VG, llvm::raw_string_ostream &Comment)
 
static MCCFIInstruction createDefCFAExpression (const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
 
static void emitFrameOffsetAdj (MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo *TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool *HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg)
 
static bool isCombineInstrSettingFlag (unsigned Opc)
 
static bool isCombineInstrCandidate32 (unsigned Opc)
 
static bool isCombineInstrCandidate64 (unsigned Opc)
 
static bool isCombineInstrCandidateFP (const MachineInstr &Inst)
 
static bool isCombineInstrCandidate (unsigned Opc)
 
static bool canCombine (MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
 
static bool canCombineWithMUL (MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc, unsigned ZeroReg)
 
static bool canCombineWithFMUL (MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc)
 
static bool getMaddPatterns (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
 Find instructions that can be turned into madd.
 
static bool getFMAPatterns (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
 Floating-Point Support.
 
static bool getFMULPatterns (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
 
static bool getFNEGPatterns (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
 
static bool getMiscPatterns (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
 Find other MI combine patterns.
 
static MachineInstrgenFusedMultiply (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC, FMAInstKind kind=FMAInstKind::Default, const Register *ReplacedAddend=nullptr)
 genFusedMultiply - Generate fused multiply instructions.
 
static MachineInstrgenFNegatedMAD (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs)
 
static MachineInstrgenIndexedMultiply (MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxDupOp, unsigned MulOpc, const TargetRegisterClass *RC, MachineRegisterInfo &MRI)
 Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
 
static MachineInstrgenFusedMultiplyAcc (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
 genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.
 
static Register genNeg (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned MnegOpc, const TargetRegisterClass *RC)
 genNeg - Helper to generate an intermediate negation of the second operand of Root
 
static MachineInstrgenFusedMultiplyAccNeg (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
 genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional negation of the accumulator
 
static MachineInstrgenFusedMultiplyIdx (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
 genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.
 
static MachineInstrgenFusedMultiplyIdxNeg (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
 genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional negation of the accumulator
 
static MachineInstrgenMaddR (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR, const TargetRegisterClass *RC)
 genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example - an ADD intermediate needs to be stored in a register: MUL I=A,B,0 ADD R,I,Imm ==> ORR V, ZR, Imm ==> MADD R,A,B,V
 
static void genSubAdd2SubSub (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo *TII, MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, unsigned IdxOpd1, DenseMap< unsigned, unsigned > &InstrIdxForVirtReg)
 Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.
 
static bool outliningCandidatesSigningScopeConsensus (const outliner::Candidate &a, const outliner::Candidate &b)
 
static bool outliningCandidatesSigningKeyConsensus (const outliner::Candidate &a, const outliner::Candidate &b)
 
static bool outliningCandidatesV8_3OpsConsensus (const outliner::Candidate &a, const outliner::Candidate &b)
 
static void signOutlinedFunction (MachineFunction &MF, MachineBasicBlock &MBB, const AArch64InstrInfo *TII, bool ShouldSignReturnAddr)
 
static std::optional< ParamLoadedValuedescribeORRLoadedValue (const MachineInstr &MI, Register DescribedReg, const TargetInstrInfo *TII, const TargetRegisterInfo *TRI)
 If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then, if possible, describe the value in terms of the source register.
 
static Register cloneInstr (const MachineInstr *MI, unsigned ReplaceOprNum, Register ReplaceReg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertTo)
 Clone an instruction from MI.
 
static void extractPhiReg (const MachineInstr &Phi, const MachineBasicBlock *MBB, Register &RegMBB, Register &RegOther)
 
static bool isDefinedOutside (Register Reg, const MachineBasicBlock *BB)
 
static bool getIndVarInfo (Register Reg, const MachineBasicBlock *LoopBB, MachineInstr *&UpdateInst, unsigned &UpdateCounterOprNum, Register &InitReg, bool &IsUpdatePriorComp)
 If Reg is an induction variable, return true and set some parameters.
 

Variables

static cl::opt< unsignedTBZDisplacementBits ("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"))
 
static cl::opt< unsignedCBZDisplacementBits ("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"))
 
static cl::opt< unsignedBCCDisplacementBits ("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)"))
 
static cl::opt< unsignedBDisplacementBits ("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)"))
 

Macro Definition Documentation

◆ GET_INSTRINFO_CTOR_DTOR

#define GET_INSTRINFO_CTOR_DTOR

Definition at line 63 of file AArch64InstrInfo.cpp.

◆ GET_INSTRINFO_HELPERS

#define GET_INSTRINFO_HELPERS

Definition at line 10253 of file AArch64InstrInfo.cpp.

◆ GET_INSTRMAP_INFO

#define GET_INSTRMAP_INFO

Definition at line 10254 of file AArch64InstrInfo.cpp.

Enumeration Type Documentation

◆ AccessKind

enum AccessKind
Enumerator
AK_Write 
AK_Read 
AK_All 

Definition at line 1324 of file AArch64InstrInfo.cpp.

◆ FMAInstKind

enum class FMAInstKind
strong
Enumerator
Default 
Indexed 
Accumulator 

Definition at line 6854 of file AArch64InstrInfo.cpp.

◆ MachineOutlinerClass

Constants defining how certain sequences should be outlined.

This encompasses how an outlined function should be called, and what kind of frame should be emitted for that outlined function.

MachineOutlinerDefault implies that the function should be called with a save and restore of LR to the stack.

That is,

I1 Save LR OUTLINED_FUNCTION: I2 --> BL OUTLINED_FUNCTION I1 I3 Restore LR I2 I3 RET

  • Call construction overhead: 3 (save + BL + restore)
  • Frame construction overhead: 1 (ret)
  • Requires stack fixups? Yes

MachineOutlinerTailCall implies that the function is being created from a sequence of instructions ending in a return.

That is,

I1 OUTLINED_FUNCTION: I2 --> B OUTLINED_FUNCTION I1 RET I2 RET

  • Call construction overhead: 1 (B)
  • Frame construction overhead: 0 (Return included in sequence)
  • Requires stack fixups? No

MachineOutlinerNoLRSave implies that the function should be called using a BL instruction, but doesn't require LR to be saved and restored. This happens when LR is known to be dead.

That is,

I1 OUTLINED_FUNCTION: I2 --> BL OUTLINED_FUNCTION I1 I3 I2 I3 RET

  • Call construction overhead: 1 (BL)
  • Frame construction overhead: 1 (RET)
  • Requires stack fixups? No

MachineOutlinerThunk implies that the function is being created from a sequence of instructions ending in a call. The outlined function is called with a BL instruction, and the outlined function tail-calls the original call destination.

That is,

I1 OUTLINED_FUNCTION: I2 --> BL OUTLINED_FUNCTION I1 BL f I2 B f

  • Call construction overhead: 1 (BL)
  • Frame construction overhead: 0
  • Requires stack fixups? No

MachineOutlinerRegSave implies that the function should be called with a save and restore of LR to an available register. This allows us to avoid stack fixups. Note that this outlining variant is compatible with the NoLRSave case.

That is,

I1 Save LR OUTLINED_FUNCTION: I2 --> BL OUTLINED_FUNCTION I1 I3 Restore LR I2 I3 RET

  • Call construction overhead: 3 (save + BL + restore)
  • Frame construction overhead: 1 (ret)
  • Requires stack fixups? No
Enumerator
MachineOutlinerDefault 
MachineOutlinerTailCall 

Emit a save, restore, call, and return.

MachineOutlinerNoLRSave 

Only emit a branch.

MachineOutlinerThunk 

Emit a call and return.

MachineOutlinerRegSave 

Emit a call and tail-call.

Same as default, but save to a register.

MachineOutlinerTailCall 
MachineOutlinerThunk 
MachineOutlinerNoLRSave 
MachineOutlinerRegSave 
MachineOutlinerDefault 
MachineOutlinerDefault 
MachineOutlinerTailCall 

Definition at line 8439 of file AArch64InstrInfo.cpp.

◆ MachineOutlinerMBBFlags

Enumerator
LRUnavailableSomewhere 
HasCalls 
UnsafeRegsDead 
LRUnavailableSomewhere 
HasCalls 
UnsafeRegsDead 

Definition at line 8447 of file AArch64InstrInfo.cpp.

Function Documentation

◆ AddSubReg()

static const MachineInstrBuilder & AddSubReg ( const MachineInstrBuilder MIB,
unsigned  Reg,
unsigned  SubIdx,
unsigned  State,
const TargetRegisterInfo TRI 
)
static

◆ appendVGScaledOffsetExpr()

static void appendVGScaledOffsetExpr ( SmallVectorImpl< char > &  Expr,
int  NumBytes,
int  NumVGScaledBytes,
unsigned  VG,
llvm::raw_string_ostream Comment 
)
static

◆ areCFlagsAccessedBetweenInstrs()

static bool areCFlagsAccessedBetweenInstrs ( MachineBasicBlock::iterator  From,
MachineBasicBlock::iterator  To,
const TargetRegisterInfo TRI,
const AccessKind  AccessToCheck = AK_All 
)
static

True when condition flags are accessed (either by writing or reading) on the instruction trace starting at From and ending at To.

Note: If From and To are from different blocks it's assumed CC are accessed on the path.

Definition at line 1331 of file AArch64InstrInfo.cpp.

References AK_Read, AK_Write, assert(), From, llvm::MachineInstrBundleIterator< Ty, IsReverse >::getReverse(), llvm::instructionsWithoutDebug(), MI, and TRI.

Referenced by canCmpInstrBeRemoved(), canInstrSubstituteCmpInstr(), and llvm::AArch64InstrInfo::optimizeCondBranch().

◆ areCFlagsAliveInSuccessors()

static bool areCFlagsAliveInSuccessors ( const MachineBasicBlock MBB)
static

Check if AArch64::NZCV should be alive in successors of MBB.

Definition at line 1631 of file AArch64InstrInfo.cpp.

References MBB, and llvm::MachineBasicBlock::successors().

Referenced by llvm::examineCFlagsUse().

◆ canCmpInstrBeRemoved()

static bool canCmpInstrBeRemoved ( MachineInstr MI,
MachineInstr CmpInstr,
int  CmpValue,
const TargetRegisterInfo TRI,
SmallVectorImpl< MachineInstr * > &  CCUseInstrs,
bool IsInvertCC 
)
static

◆ canCombine()

static bool canCombine ( MachineBasicBlock MBB,
MachineOperand MO,
unsigned  CombineOpc,
unsigned  ZeroReg = 0,
bool  CheckZeroReg = false 
)
static

◆ canCombineWithFMUL()

static bool canCombineWithFMUL ( MachineBasicBlock MBB,
MachineOperand MO,
unsigned  MulOpc 
)
static

Definition at line 6181 of file AArch64InstrInfo.cpp.

References canCombine(), and MBB.

Referenced by getFMAPatterns().

◆ canCombineWithMUL()

static bool canCombineWithMUL ( MachineBasicBlock MBB,
MachineOperand MO,
unsigned  MulOpc,
unsigned  ZeroReg 
)
static

Definition at line 6174 of file AArch64InstrInfo.cpp.

References canCombine(), and MBB.

Referenced by getMaddPatterns().

◆ canFoldIntoCSel()

static unsigned canFoldIntoCSel ( const MachineRegisterInfo MRI,
unsigned  VReg,
unsigned NewVReg = nullptr 
)
static

◆ canInstrSubstituteCmpInstr()

static bool canInstrSubstituteCmpInstr ( MachineInstr MI,
MachineInstr CmpInstr,
const TargetRegisterInfo TRI 
)
static

Check if CmpInstr can be substituted by MI.

CmpInstr can be substituted:

  • CmpInstr is either 'ADDS vreg, 0' or 'SUBS vreg, 0'
  • and, MI and CmpInstr are from the same MachineBB
  • and, condition flags are not alive in successors of the CmpInstr parent
  • and, if MI opcode is the S form there must be no defs of flags between MI and CmpInstr or if MI opcode is not the S form there must be neither defs of flags nor uses of flags between MI and CmpInstr.
  • and, if C/V flags are not used after CmpInstr or if N flag is used but MI produces poison value if signed overflow occurs.

Definition at line 1777 of file AArch64InstrInfo.cpp.

References AK_All, AK_Write, areCFlagsAccessedBetweenInstrs(), assert(), llvm::examineCFlagsUse(), llvm::MachineOperand::getImm(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), isADDSRegImm(), llvm::MachineOperand::isImm(), isSUBSRegImm(), MI, llvm::MachineInstr::NoSWrap, sForm(), and TRI.

◆ canPairLdStOpc()

static bool canPairLdStOpc ( unsigned  FirstOpc,
unsigned  SecondOpc 
)
static

Definition at line 4451 of file AArch64InstrInfo.cpp.

Referenced by llvm::AArch64InstrInfo::shouldClusterMemOps().

◆ cloneInstr()

static Register cloneInstr ( const MachineInstr MI,
unsigned  ReplaceOprNum,
Register  ReplaceReg,
MachineBasicBlock MBB,
MachineBasicBlock::iterator  InsertTo 
)
static

◆ convertToNonFlagSettingOpc()

static unsigned convertToNonFlagSettingOpc ( const MachineInstr MI)
static

Return the opcode that does not set flags when possible - otherwise return the original opcode.

The caller is responsible to do the actual substitution and legality checking.

Definition at line 1278 of file AArch64InstrInfo.cpp.

References MI.

Referenced by getMaddPatterns(), and llvm::AArch64InstrInfo::optimizeCompareInstr().

◆ createDefCFAExpression()

static MCCFIInstruction createDefCFAExpression ( const TargetRegisterInfo TRI,
unsigned  Reg,
const StackOffset Offset 
)
static

◆ describeORRLoadedValue()

static std::optional< ParamLoadedValue > describeORRLoadedValue ( const MachineInstr MI,
Register  DescribedReg,
const TargetInstrInfo TII,
const TargetRegisterInfo TRI 
)
static

If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then, if possible, describe the value in terms of the source register.

Definition at line 9579 of file AArch64InstrInfo.cpp.

References assert(), llvm::MachineOperand::CreateReg(), llvm::MDNode::get(), MI, TII, and TRI.

Referenced by llvm::AArch64InstrInfo::describeLoadedValue().

◆ emitFrameOffsetAdj()

static void emitFrameOffsetAdj ( MachineBasicBlock MBB,
MachineBasicBlock::iterator  MBBI,
const DebugLoc DL,
unsigned  DestReg,
unsigned  SrcReg,
int64_t  Offset,
unsigned  Opc,
const TargetInstrInfo TII,
MachineInstr::MIFlag  Flag,
bool  NeedsWinCFI,
bool HasWinCFI,
bool  EmitCFAOffset,
StackOffset  CFAOffset,
unsigned  FrameReg 
)
static

◆ extractPhiReg()

static void extractPhiReg ( const MachineInstr Phi,
const MachineBasicBlock MBB,
Register RegMBB,
Register RegOther 
)
static

Definition at line 10045 of file AArch64InstrInfo.cpp.

References assert(), and MBB.

Referenced by getIndVarInfo().

◆ findCondCodeUsedByInstr()

static AArch64CC::CondCode findCondCodeUsedByInstr ( const MachineInstr Instr)
static

Find a condition code used by the instruction.

Returns AArch64CC::Invalid if either the instruction does not use condition codes or we don't optimize CmpInstr in the presence of such instructions.

Definition at line 1672 of file AArch64InstrInfo.cpp.

References findCondCodeUseOperandIdxForBranchOrSelect(), and llvm::AArch64CC::Invalid.

Referenced by canCmpInstrBeRemoved(), and llvm::examineCFlagsUse().

◆ findCondCodeUseOperandIdxForBranchOrSelect()

static int findCondCodeUseOperandIdxForBranchOrSelect ( const MachineInstr Instr)
static
Returns
The condition code operand index for Instr if it is a branch or select and -1 otherwise.

Definition at line 1641 of file AArch64InstrInfo.cpp.

References assert(), and Idx.

Referenced by findCondCodeUsedByInstr().

◆ forwardCopyWillClobberTuple()

static bool forwardCopyWillClobberTuple ( unsigned  DestReg,
unsigned  SrcReg,
unsigned  NumRegs 
)
static

◆ genFNegatedMAD()

static MachineInstr * genFNegatedMAD ( MachineFunction MF,
MachineRegisterInfo MRI,
const TargetInstrInfo TII,
MachineInstr Root,
SmallVectorImpl< MachineInstr * > &  InsInstrs 
)
static

◆ genFusedMultiply()

static MachineInstr * genFusedMultiply ( MachineFunction MF,
MachineRegisterInfo MRI,
const TargetInstrInfo TII,
MachineInstr Root,
SmallVectorImpl< MachineInstr * > &  InsInstrs,
unsigned  IdxMulOpd,
unsigned  MaddOpc,
const TargetRegisterClass RC,
FMAInstKind  kind = FMAInstKind::Default,
const Register ReplacedAddend = nullptr 
)
static

genFusedMultiply - Generate fused multiply instructions.

This function supports both integer and floating point instructions. A typical example: F|MUL I=A,B,0 F|ADD R,I,C ==> F|MADD R,A,B,C

Parameters
MFContaining MachineFunction
MRIRegister information
TIITarget information
Rootis the F|ADD instruction
[out]InsInstrsis a vector of machine instructions and will contain the generated madd instruction
IdxMulOpdis index of operand in Root that is the result of the F|MUL. In the example above IdxMulOpd is 1.
MaddOpcthe opcode fo the f|madd instruction
RCRegister class of operands
kindof fma instruction (addressing mode) to be generated
ReplacedAddendis the result register from the instruction replacing the non-combined operand, if any.

Definition at line 6875 of file AArch64InstrInfo.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::getKillRegState(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::MachineOperand::isKill(), llvm::Register::isVirtual(), MRI, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and TII.

Referenced by llvm::AArch64InstrInfo::genAlternativeCodeSequence(), genFusedMultiplyAcc(), genFusedMultiplyAccNeg(), genFusedMultiplyIdx(), and genFusedMultiplyIdxNeg().

◆ genFusedMultiplyAcc()

static MachineInstr * genFusedMultiplyAcc ( MachineFunction MF,
MachineRegisterInfo MRI,
const TargetInstrInfo TII,
MachineInstr Root,
SmallVectorImpl< MachineInstr * > &  InsInstrs,
unsigned  IdxMulOpd,
unsigned  MaddOpc,
const TargetRegisterClass RC 
)
static

genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.

See also
genFusedMultiply

Definition at line 7019 of file AArch64InstrInfo.cpp.

References genFusedMultiply(), MRI, and TII.

Referenced by llvm::AArch64InstrInfo::genAlternativeCodeSequence().

◆ genFusedMultiplyAccNeg()

static MachineInstr * genFusedMultiplyAccNeg ( MachineFunction MF,
MachineRegisterInfo MRI,
const TargetInstrInfo TII,
MachineInstr Root,
SmallVectorImpl< MachineInstr * > &  InsInstrs,
DenseMap< unsigned, unsigned > &  InstrIdxForVirtReg,
unsigned  IdxMulOpd,
unsigned  MaddOpc,
unsigned  MnegOpc,
const TargetRegisterClass RC 
)
static

genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional negation of the accumulator

Definition at line 7048 of file AArch64InstrInfo.cpp.

References assert(), genFusedMultiply(), genNeg(), MRI, and TII.

Referenced by llvm::AArch64InstrInfo::genAlternativeCodeSequence().

◆ genFusedMultiplyIdx()

static MachineInstr * genFusedMultiplyIdx ( MachineFunction MF,
MachineRegisterInfo MRI,
const TargetInstrInfo TII,
MachineInstr Root,
SmallVectorImpl< MachineInstr * > &  InsInstrs,
unsigned  IdxMulOpd,
unsigned  MaddOpc,
const TargetRegisterClass RC 
)
static

genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.

See also
genFusedMultiply

Definition at line 7065 of file AArch64InstrInfo.cpp.

References genFusedMultiply(), MRI, and TII.

Referenced by llvm::AArch64InstrInfo::genAlternativeCodeSequence().

◆ genFusedMultiplyIdxNeg()

static MachineInstr * genFusedMultiplyIdxNeg ( MachineFunction MF,
MachineRegisterInfo MRI,
const TargetInstrInfo TII,
MachineInstr Root,
SmallVectorImpl< MachineInstr * > &  InsInstrs,
DenseMap< unsigned, unsigned > &  InstrIdxForVirtReg,
unsigned  IdxMulOpd,
unsigned  MaddOpc,
unsigned  MnegOpc,
const TargetRegisterClass RC 
)
static

genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional negation of the accumulator

Definition at line 7075 of file AArch64InstrInfo.cpp.

References assert(), genFusedMultiply(), genNeg(), MRI, and TII.

Referenced by llvm::AArch64InstrInfo::genAlternativeCodeSequence().

◆ genIndexedMultiply()

static MachineInstr * genIndexedMultiply ( MachineInstr Root,
SmallVectorImpl< MachineInstr * > &  InsInstrs,
unsigned  IdxDupOp,
unsigned  MulOpc,
const TargetRegisterClass RC,
MachineRegisterInfo MRI 
)
static

◆ genMaddR()

static MachineInstr * genMaddR ( MachineFunction MF,
MachineRegisterInfo MRI,
const TargetInstrInfo TII,
MachineInstr Root,
SmallVectorImpl< MachineInstr * > &  InsInstrs,
unsigned  IdxMulOpd,
unsigned  MaddOpc,
unsigned  VR,
const TargetRegisterClass RC 
)
static

genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example - an ADD intermediate needs to be stored in a register: MUL I=A,B,0 ADD R,I,Imm ==> ORR V, ZR, Imm ==> MADD R,A,B,V

Parameters
MFContaining MachineFunction
MRIRegister information
TIITarget information
Rootis the ADD instruction
[out]InsInstrsis a vector of machine instructions and will contain the generated madd instruction
IdxMulOpdis index of operand in Root that is the result of the MUL. In the example above IdxMulOpd is 1.
MaddOpcthe opcode fo the madd instruction
VRis a virtual register that holds the value of an ADD operand (V in the example above).
RCRegister class of operands

Definition at line 7108 of file AArch64InstrInfo.cpp.

References llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::getKillRegState(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::Register::isVirtual(), llvm::Register::isVirtualRegister(), MRI, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and TII.

Referenced by llvm::AArch64InstrInfo::genAlternativeCodeSequence().

◆ genNeg()

static Register genNeg ( MachineFunction MF,
MachineRegisterInfo MRI,
const TargetInstrInfo TII,
MachineInstr Root,
SmallVectorImpl< MachineInstr * > &  InsInstrs,
DenseMap< unsigned, unsigned > &  InstrIdxForVirtReg,
unsigned  MnegOpc,
const TargetRegisterClass RC 
)
static

◆ genSubAdd2SubSub()

static void genSubAdd2SubSub ( MachineFunction MF,
MachineRegisterInfo MRI,
const TargetInstrInfo TII,
MachineInstr Root,
SmallVectorImpl< MachineInstr * > &  InsInstrs,
SmallVectorImpl< MachineInstr * > &  DelInstrs,
unsigned  IdxOpd1,
DenseMap< unsigned, unsigned > &  InstrIdxForVirtReg 
)
static

◆ getBranchDisplacementBits()

static unsigned getBranchDisplacementBits ( unsigned  Opc)
static

◆ getFMAPatterns()

static bool getFMAPatterns ( MachineInstr Root,
SmallVectorImpl< unsigned > &  Patterns 
)
static

◆ getFMULPatterns()

static bool getFMULPatterns ( MachineInstr Root,
SmallVectorImpl< unsigned > &  Patterns 
)
static

◆ getFNEGPatterns()

static bool getFNEGPatterns ( MachineInstr Root,
SmallVectorImpl< unsigned > &  Patterns 
)
static

◆ getIndVarInfo()

static bool getIndVarInfo ( Register  Reg,
const MachineBasicBlock LoopBB,
MachineInstr *&  UpdateInst,
unsigned UpdateCounterOprNum,
Register InitReg,
bool IsUpdatePriorComp 
)
static

◆ getMaddPatterns()

static bool getMaddPatterns ( MachineInstr Root,
SmallVectorImpl< unsigned > &  Patterns 
)
static

◆ getMiscPatterns()

static bool getMiscPatterns ( MachineInstr Root,
SmallVectorImpl< unsigned > &  Patterns 
)
static

◆ getRegClass()

static const TargetRegisterClass * getRegClass ( const MachineInstr MI,
Register  Reg 
)
static

◆ getUsedNZCV()

static UsedNZCV getUsedNZCV ( AArch64CC::CondCode  CC)
static

◆ isADDSRegImm()

static bool isADDSRegImm ( unsigned  Opcode)
static

Definition at line 1756 of file AArch64InstrInfo.cpp.

Referenced by canCmpInstrBeRemoved(), and canInstrSubstituteCmpInstr().

◆ isCheapImmediate()

static bool isCheapImmediate ( const MachineInstr MI,
unsigned  BitSize 
)
static

◆ isCombineInstrCandidate()

static bool isCombineInstrCandidate ( unsigned  Opc)
static

Definition at line 6134 of file AArch64InstrInfo.cpp.

References isCombineInstrCandidate32(), and isCombineInstrCandidate64().

Referenced by getMaddPatterns().

◆ isCombineInstrCandidate32()

static bool isCombineInstrCandidate32 ( unsigned  Opc)
static

Definition at line 6052 of file AArch64InstrInfo.cpp.

Referenced by isCombineInstrCandidate().

◆ isCombineInstrCandidate64()

static bool isCombineInstrCandidate64 ( unsigned  Opc)
static

Definition at line 6071 of file AArch64InstrInfo.cpp.

Referenced by isCombineInstrCandidate().

◆ isCombineInstrCandidateFP()

static bool isCombineInstrCandidateFP ( const MachineInstr Inst)
static

◆ isCombineInstrSettingFlag()

static bool isCombineInstrSettingFlag ( unsigned  Opc)
static

Definition at line 6033 of file AArch64InstrInfo.cpp.

Referenced by canCombine(), getMaddPatterns(), and getMiscPatterns().

◆ isDefinedOutside()

static bool isDefinedOutside ( Register  Reg,
const MachineBasicBlock BB 
)
static

◆ isPostIndexLdStOpcode()

static bool isPostIndexLdStOpcode ( unsigned  Opcode)
static

Return true if the opcode is a post-index ld/st instruction, which really loads from base+0.

Definition at line 3492 of file AArch64InstrInfo.cpp.

Referenced by llvm::AArch64InstrInfo::getMemOperandWithOffsetWidth().

◆ isSUBSRegImm()

static bool isSUBSRegImm ( unsigned  Opcode)
static

Definition at line 1760 of file AArch64InstrInfo.cpp.

Referenced by canCmpInstrBeRemoved(), and canInstrSubstituteCmpInstr().

◆ loadRegPairFromStackSlot()

static void loadRegPairFromStackSlot ( const TargetRegisterInfo TRI,
MachineBasicBlock MBB,
MachineBasicBlock::iterator  InsertBefore,
const MCInstrDesc MCID,
Register  DestReg,
unsigned  SubIdx0,
unsigned  SubIdx1,
int  FI,
MachineMemOperand MMO 
)
static

◆ offsetExtendOpcode()

static unsigned offsetExtendOpcode ( unsigned  Opcode)
static

Definition at line 3321 of file AArch64InstrInfo.cpp.

References llvm_unreachable.

Referenced by llvm::AArch64InstrInfo::emitLdStWithAddr().

◆ outliningCandidatesSigningKeyConsensus()

static bool outliningCandidatesSigningKeyConsensus ( const outliner::Candidate a,
const outliner::Candidate b 
)
static

◆ outliningCandidatesSigningScopeConsensus()

static bool outliningCandidatesSigningScopeConsensus ( const outliner::Candidate a,
const outliner::Candidate b 
)
static

◆ outliningCandidatesV8_3OpsConsensus()

static bool outliningCandidatesV8_3OpsConsensus ( const outliner::Candidate a,
const outliner::Candidate b 
)
static

◆ parseCondBranch()

static void parseCondBranch ( MachineInstr LastInst,
MachineBasicBlock *&  Target,
SmallVectorImpl< MachineOperand > &  Cond 
)
static

◆ regOffsetOpcode()

static unsigned regOffsetOpcode ( unsigned  Opcode)
static

Definition at line 3066 of file AArch64InstrInfo.cpp.

References llvm_unreachable.

Referenced by llvm::AArch64InstrInfo::emitLdStWithAddr().

◆ removeCopies()

static unsigned removeCopies ( const MachineRegisterInfo MRI,
unsigned  VReg 
)
static

◆ scaledOffsetOpcode()

unsigned scaledOffsetOpcode ( unsigned  Opcode,
unsigned Scale 
)

Definition at line 3140 of file AArch64InstrInfo.cpp.

References llvm_unreachable.

Referenced by llvm::AArch64InstrInfo::emitLdStWithAddr().

◆ scaleOffset()

static bool scaleOffset ( unsigned  Opc,
int64_t &  Offset 
)
static

◆ sForm()

static unsigned sForm ( MachineInstr Instr)
static

Get opcode of S version of Instr.

If Instr is S version its opcode is returned. AArch64::INSTRUCTION_LIST_END is returned if Instr does not have S version or we are not interested in it.

Definition at line 1584 of file AArch64InstrInfo.cpp.

Referenced by canInstrSubstituteCmpInstr().

◆ shouldClusterFI()

static bool shouldClusterFI ( const MachineFrameInfo MFI,
int  FI1,
int64_t  Offset1,
unsigned  Opcode1,
int  FI2,
int64_t  Offset2,
unsigned  Opcode2 
)
static

◆ signOutlinedFunction()

static void signOutlinedFunction ( MachineFunction MF,
MachineBasicBlock MBB,
const AArch64InstrInfo TII,
bool  ShouldSignReturnAddr 
)
static

◆ storeRegPairToStackSlot()

static void storeRegPairToStackSlot ( const TargetRegisterInfo TRI,
MachineBasicBlock MBB,
MachineBasicBlock::iterator  InsertBefore,
const MCInstrDesc MCID,
Register  SrcReg,
bool  IsKill,
unsigned  SubIdx0,
unsigned  SubIdx1,
int  FI,
MachineMemOperand MMO 
)
static

◆ unscaledOffsetOpcode()

unsigned unscaledOffsetOpcode ( unsigned  Opcode)

Definition at line 3245 of file AArch64InstrInfo.cpp.

References llvm_unreachable.

Referenced by llvm::AArch64InstrInfo::emitLdStWithAddr().

◆ UpdateOperandRegClass()

static bool UpdateOperandRegClass ( MachineInstr Instr)
static

Variable Documentation

◆ BCCDisplacementBits

cl::opt< unsigned > BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)")) ( "aarch64-bcc-offset-bits"  ,
cl::Hidden  ,
cl::init(19)  ,
cl::desc("Restrict range of Bcc instructions (DEBUG)")   
)
static

◆ BDisplacementBits

cl::opt< unsigned > BDisplacementBits("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)")) ( "aarch64-b-offset-bits"  ,
cl::Hidden  ,
cl::init(26)  ,
cl::desc("Restrict range of B instructions (DEBUG)")   
)
static

◆ CBZDisplacementBits

cl::opt< unsigned > CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)")) ( "aarch64-cbz-offset-bits"  ,
cl::Hidden  ,
cl::init(19)  ,
cl::desc("Restrict range of CB[N]Z instructions (DEBUG)")   
)
static

◆ TBZDisplacementBits

cl::opt< unsigned > TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)")) ( "aarch64-tbz-offset-bits"  ,
cl::Hidden  ,
cl::init(14)  ,
cl::desc("Restrict range of TB[N]Z instructions (DEBUG)")   
)
static