AArch64LoadStoreOptimizer.cpp File Reference

#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCDwarf.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include <cassert>
#include <cstdint>
#include <functional>
#include <iterator>
#include <limits>
#include <optional>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "aarch64-ldst-opt"
#define	AARCH64_LOAD_STORE_OPT_NAME   "AArch64 load / store optimization pass"

Functions
	STATISTIC (NumPairCreated, "Number of load/store pair instructions generated")
	STATISTIC (NumPostFolded, "Number of post-index updates folded")
	STATISTIC (NumPreFolded, "Number of pre-index updates folded")
	STATISTIC (NumUnscaledPairCreated, "Number of load/store from unscaled generated")
	STATISTIC (NumZeroStoresPromoted, "Number of narrow zero stores promoted")
	STATISTIC (NumLoadsFromStoresPromoted, "Number of loads from stores promoted")
	STATISTIC (NumFailedAlignmentCheck, "Number of load/store pair transformation " "not passed the alignment check")
	STATISTIC (NumConstOffsetFolded, "Number of const offset of index address folded")
	DEBUG_COUNTER (RegRenamingCounter, DEBUG_TYPE "-reg-renaming", "Controls which pairs are considered for renaming")
	INITIALIZE_PASS (AArch64LoadStoreOpt, "aarch64-ldst-opt", AARCH64_LOAD_STORE_OPT_NAME, false, false) static bool isNarrowStore(unsigned Opc)
static bool	isTagStore (const MachineInstr &MI)
static unsigned	getMatchingNonSExtOpcode (unsigned Opc, bool *IsValidLdStrOpc=nullptr)
static unsigned	getMatchingWideOpcode (unsigned Opc)
static unsigned	getMatchingPairOpcode (unsigned Opc)
static unsigned	isMatchingStore (MachineInstr &LoadInst, MachineInstr &StoreInst)
static unsigned	getPreIndexedOpcode (unsigned Opc)
static unsigned	getBaseAddressOpcode (unsigned Opc)
static unsigned	getPostIndexedOpcode (unsigned Opc)
static bool	isPreLdStPairCandidate (MachineInstr &FirstMI, MachineInstr &MI)
static void	getPrePostIndexedMemOpInfo (const MachineInstr &MI, int &Scale, int &MinOffset, int &MaxOffset)
static MachineOperand &	getLdStRegOp (MachineInstr &MI, unsigned PairedRegOp=0)
static bool	isLdOffsetInRangeOfSt (MachineInstr &LoadInst, MachineInstr &StoreInst, const AArch64InstrInfo *TII)
static bool	isPromotableZeroStoreInst (MachineInstr &MI)
static bool	isPromotableLoadFromStore (MachineInstr &MI)
static bool	isMergeableLdStUpdate (MachineInstr &MI, AArch64FunctionInfo &AFI)
static bool	isMergeableIndexLdSt (MachineInstr &MI, int &Scale)
static bool	isRewritableImplicitDef (unsigned Opc)
static bool	forAllMIsUntilDef (MachineInstr &MI, MCPhysReg DefReg, const TargetRegisterInfo *TRI, unsigned Limit, std::function< bool(MachineInstr &, bool)> &Fn)
static void	updateDefinedRegisters (MachineInstr &MI, LiveRegUnits &Units, const TargetRegisterInfo *TRI)
static void	addDebugSubstitutionsToTable (MachineFunction *MF, unsigned InstrNumToSet, MachineInstr &OriginalInstr, MachineInstr &MergedInstr)
	This function will add a new entry into the debugValueSubstitutions table when two instruction have been merged into a new one represented by MergedInstr.
static bool	inBoundsForPair (bool IsUnscaled, int Offset, int OffsetStride)
static int	alignTo (int Num, int PowOf2)
static bool	mayAlias (MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)
static bool	needsWinCFI (const MachineFunction *MF)
static bool	areCandidatesToMergeOrPair (MachineInstr &FirstMI, MachineInstr &MI, LdStPairFlags &Flags, const AArch64InstrInfo *TII)
static bool	canRenameMOP (const MachineOperand &MOP, const TargetRegisterInfo *TRI)
static bool	canRenameUpToDef (MachineInstr &FirstMI, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static bool	canRenameUntilSecondLoad (MachineInstr &FirstLoad, MachineInstr &SecondLoad, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static std::optional< MCPhysReg >	tryToFindRegisterToRename (const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static std::optional< MCPhysReg >	findRenameRegForSameLdStRegPair (std::optional< bool > MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static MachineBasicBlock::iterator	maybeMoveCFI (MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI)

Variables
static cl::opt< unsigned >	LdStLimit ("aarch64-load-store-scan-limit", cl::init(20), cl::Hidden)
static cl::opt< unsigned >	UpdateLimit ("aarch64-update-scan-limit", cl::init(100), cl::Hidden)
static cl::opt< unsigned >	LdStConstLimit ("aarch64-load-store-const-scan-limit", cl::init(10), cl::Hidden)
static cl::opt< bool >	EnableRenaming ("aarch64-load-store-renaming", cl::init(true), cl::Hidden)

Macro Definition Documentation

AARCH64_LOAD_STORE_OPT_NAME

#define AARCH64_LOAD_STORE_OPT_NAME   "AArch64 load / store optimization pass"

Definition at line 90 of file AArch64LoadStoreOptimizer.cpp.

DEBUG_TYPE

#define DEBUG_TYPE   "aarch64-ldst-opt"

Definition at line 55 of file AArch64LoadStoreOptimizer.cpp.

Function Documentation

addDebugSubstitutionsToTable()

static void addDebugSubstitutionsToTable ( MachineFunction *  MF, unsigned  InstrNumToSet, MachineInstr &  OriginalInstr, MachineInstr &  MergedInstr  )

static

This function will add a new entry into the debugValueSubstitutions table when two instruction have been merged into a new one represented by MergedInstr.

Definition at line 969 of file AArch64LoadStoreOptimizer.cpp.

References llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::MachineFunction::makeDebugValueSubstitution(), llvm::MachineInstr::operands(), and llvm::MachineInstr::peekDebugInstrNum().

alignTo()

static int alignTo ( int  Num, int  PowOf2  )

static

Definition at line 1544 of file AArch64LoadStoreOptimizer.cpp.

Referenced by llvm::TargetFrameLowering::alignSPAdjust().

areCandidatesToMergeOrPair()

static bool areCandidatesToMergeOrPair ( MachineInstr &  FirstMI, MachineInstr &  MI, LdStPairFlags &  Flags, const AArch64InstrInfo *  TII  )

static

Definition at line 1630 of file AArch64LoadStoreOptimizer.cpp.

References assert(), llvm::MachineInstr::FrameDestroy, llvm::MachineInstr::FrameSetup, getLdStRegOp(), getMatchingNonSExtOpcode(), getMatchingPairOpcode(), llvm::MachineInstr::getOpcode(), llvm::MachineOperand::getReg(), llvm::MachineInstr::hasOrderedMemoryRef(), llvm::AArch64InstrInfo::isPreLdSt(), isPreLdStPairCandidate(), MI, needsWinCFI(), and TII.

canRenameMOP()

static bool canRenameMOP ( const MachineOperand &  MOP, const TargetRegisterInfo *  TRI  )

static

Definition at line 1699 of file AArch64LoadStoreOptimizer.cpp.

References llvm::dbgs(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getParent(), llvm::MachineOperand::getReg(), llvm::MachineOperand::isDef(), llvm::MachineOperand::isEarlyClobber(), llvm::MachineOperand::isImplicit(), llvm::MachineOperand::isReg(), llvm::MachineOperand::isRenamable(), isRewritableImplicitDef(), llvm::MachineOperand::isTied(), LLVM_DEBUG, and TRI.

Referenced by canRenameUpToDef().

canRenameUntilSecondLoad()

static bool canRenameUntilSecondLoad ( MachineInstr &  FirstLoad, MachineInstr &  SecondLoad, LiveRegUnits &  UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &  RequiredClasses, const TargetRegisterInfo *  TRI  )

static

Definition at line 1839 of file AArch64LoadStoreOptimizer.cpp.

References llvm::LiveRegUnits::accumulate(), for(), llvm::ilist_node_impl< OptionsT >::getIterator(), getLdStRegOp(), llvm::MachineOperand::getReg(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::MachineInstr::isPseudo(), MI, Success, and TRI.

Referenced by findRenameRegForSameLdStRegPair().

canRenameUpToDef()

static bool canRenameUpToDef ( MachineInstr &  FirstMI, LiveRegUnits &  UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &  RequiredClasses, const TargetRegisterInfo *  TRI  )

static

Definition at line 1735 of file AArch64LoadStoreOptimizer.cpp.

References llvm::LiveRegUnits::accumulate(), llvm::any_of(), canRenameMOP(), llvm::dbgs(), forAllMIsUntilDef(), llvm::MachineInstr::FrameSetup, getLdStRegOp(), llvm::MachineOperand::getReg(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::MachineOperand::isDebug(), llvm::MachineOperand::isDef(), llvm::MachineOperand::isReg(), LdStLimit, LLVM_DEBUG, llvm::MachineInstr::mayStore(), MI, llvm::MachineInstr::operands(), and TRI.

Referenced by findRenameRegForSameLdStRegPair().

DEBUG_COUNTER()

DEBUG_COUNTER	(	RegRenamingCounter	,
		DEBUG_TYPE "-reg-renaming"	,
		"Controls which pairs are considered for renaming"
	)

findRenameRegForSameLdStRegPair()

static std::optional< MCPhysReg > findRenameRegForSameLdStRegPair ( std::optional< bool >  MaybeCanRename, MachineInstr &  FirstMI, MachineInstr &  MI, Register  Reg, LiveRegUnits &  DefinedInBB, LiveRegUnits &  UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &  RequiredClasses, const TargetRegisterInfo *  TRI  )

static

Definition at line 1927 of file AArch64LoadStoreOptimizer.cpp.

References canRenameUntilSecondLoad(), canRenameUpToDef(), getLdStRegOp(), llvm::MachineBasicBlock::getParent(), llvm::MachineInstr::getParent(), getReg(), llvm::MachineFunction::getRegInfo(), llvm::MachineInstr::mayLoad(), MI, llvm::DebugCounter::shouldExecute(), llvm::MachineRegisterInfo::tracksLiveness(), TRI, and tryToFindRegisterToRename().

forAllMIsUntilDef()

static bool forAllMIsUntilDef ( MachineInstr &  MI, MCPhysReg  DefReg, const TargetRegisterInfo *  TRI, unsigned  Limit, std::function< bool(MachineInstr &, bool)> &  Fn  )

static

Definition at line 932 of file AArch64LoadStoreOptimizer.cpp.

References llvm::any_of(), I, llvm::MachineBasicBlock::instr_rend(), llvm::instructionsWithoutDebug(), MBB, MI, and TRI.

Referenced by canRenameUpToDef().

getBaseAddressOpcode()

static unsigned getBaseAddressOpcode ( unsigned  Opc )

static

Definition at line 516 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable, and Opc.

getLdStRegOp()

static MachineOperand & getLdStRegOp ( MachineInstr &  MI, unsigned  PairedRegOp = 0  )

static

Definition at line 683 of file AArch64LoadStoreOptimizer.cpp.

References assert(), Idx, llvm::AArch64InstrInfo::isPairedLdSt(), llvm::AArch64InstrInfo::isPreLdSt(), and MI.

Referenced by areCandidatesToMergeOrPair(), canRenameUntilSecondLoad(), canRenameUpToDef(), findRenameRegForSameLdStRegPair(), and isPromotableZeroStoreInst().

getMatchingNonSExtOpcode()

static unsigned getMatchingNonSExtOpcode ( unsigned  Opc, bool *  IsValidLdStrOpc = nullptr  )

static

Definition at line 275 of file AArch64LoadStoreOptimizer.cpp.

References Opc.

Referenced by areCandidatesToMergeOrPair().

getMatchingPairOpcode()

static unsigned getMatchingPairOpcode ( unsigned  Opc )

static

Definition at line 349 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable, and Opc.

Referenced by areCandidatesToMergeOrPair().

getMatchingWideOpcode()

static unsigned getMatchingWideOpcode ( unsigned  Opc )

static

Definition at line 330 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable, and Opc.

getPostIndexedOpcode()

static unsigned getPostIndexedOpcode ( unsigned  Opc )

static

Definition at line 552 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable, and Opc.

getPreIndexedOpcode()

static unsigned getPreIndexedOpcode ( unsigned  Opc )

static

Definition at line 443 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable, and Opc.

getPrePostIndexedMemOpInfo()

static void getPrePostIndexedMemOpInfo ( const MachineInstr &  MI, int &  Scale, int &  MinOffset, int &  MaxOffset  )

static

Definition at line 665 of file AArch64LoadStoreOptimizer.cpp.

References llvm::AArch64InstrInfo::getMemScale(), llvm::AArch64InstrInfo::isPairedLdSt(), isTagStore(), and MI.

inBoundsForPair()

static bool inBoundsForPair ( bool  IsUnscaled, int  Offset, int  OffsetStride  )

static

Definition at line 1527 of file AArch64LoadStoreOptimizer.cpp.

References llvm::Offset.

INITIALIZE_PASS()

INITIALIZE_PASS	(	AArch64LoadStoreOpt	,
		"aarch64-ldst-opt"	,
		AARCH64_LOAD_STORE_OPT_NAME	,
		false	,
		false
	)

Definition at line 246 of file AArch64LoadStoreOptimizer.cpp.

References Opc.

isLdOffsetInRangeOfSt()

static bool isLdOffsetInRangeOfSt ( MachineInstr &  LoadInst, MachineInstr &  StoreInst, const AArch64InstrInfo *  TII  )

static

Definition at line 694 of file AArch64LoadStoreOptimizer.cpp.

References assert(), llvm::MachineOperand::getImm(), llvm::AArch64InstrInfo::getLdStOffsetOp(), isMatchingStore(), and TII.

isMatchingStore()

static unsigned isMatchingStore ( MachineInstr &  LoadInst, MachineInstr &  StoreInst  )

static

Definition at line 413 of file AArch64LoadStoreOptimizer.cpp.

References llvm::Instruction::getOpcode(), and llvm_unreachable.

Referenced by isLdOffsetInRangeOfSt().

isMergeableIndexLdSt()

static bool isMergeableIndexLdSt ( MachineInstr &  MI, int &  Scale  )

static

Definition at line 803 of file AArch64LoadStoreOptimizer.cpp.

References MI, and Opc.

isMergeableLdStUpdate()

static bool isMergeableLdStUpdate ( MachineInstr &  MI, AArch64FunctionInfo &  AFI  )

static

Definition at line 737 of file AArch64LoadStoreOptimizer.cpp.

References llvm::AArch64InstrInfo::getLdStBaseOp(), llvm::AArch64InstrInfo::getLdStOffsetOp(), llvm::MachineOperand::getReg(), llvm::AArch64FunctionInfo::isMTETagged(), MI, and Opc.

isPreLdStPairCandidate()

static bool isPreLdStPairCandidate ( MachineInstr &  FirstMI, MachineInstr &  MI  )

static

Definition at line 631 of file AArch64LoadStoreOptimizer.cpp.

References llvm::MachineInstr::getOpcode(), and MI.

Referenced by areCandidatesToMergeOrPair().

isPromotableLoadFromStore()

static bool isPromotableLoadFromStore ( MachineInstr &  MI )

static

Definition at line 719 of file AArch64LoadStoreOptimizer.cpp.

References MI.

isPromotableZeroStoreInst()

static bool isPromotableZeroStoreInst ( MachineInstr &  MI )

static

Definition at line 712 of file AArch64LoadStoreOptimizer.cpp.

References getLdStRegOp(), llvm::MachineOperand::getReg(), MI, and Opc.

isRewritableImplicitDef()

static bool isRewritableImplicitDef ( unsigned  Opc )

static

Definition at line 837 of file AArch64LoadStoreOptimizer.cpp.

References Opc.

Referenced by canRenameMOP().

isTagStore()

static bool isTagStore ( const MachineInstr &  MI )

static

Definition at line 263 of file AArch64LoadStoreOptimizer.cpp.

References MI.

Referenced by getPrePostIndexedMemOpInfo().

mayAlias()

static bool mayAlias ( MachineInstr &  MIa, SmallVectorImpl< MachineInstr * > &  MemInsns, AliasAnalysis *  AA  )

static

Definition at line 1548 of file AArch64LoadStoreOptimizer.cpp.

References llvm::dbgs(), LLVM_DEBUG, and llvm::MachineInstr::mayAlias().

Referenced by llvm::HexagonInstrInfo::expandPostRAPseudo().

maybeMoveCFI()

static MachineBasicBlock::iterator maybeMoveCFI ( MachineInstr &  MI, MachineBasicBlock::iterator  MaybeCFI  )

static

Definition at line 2233 of file AArch64LoadStoreOptimizer.cpp.

References assert(), End, llvm::MachineInstr::FrameDestroy, llvm::MachineInstr::FrameSetup, llvm::MachineFunction::getFrameInstructions(), llvm::MCCFIInstruction::getOperation(), MI, llvm::MCCFIInstruction::OpDefCfa, and llvm::MCCFIInstruction::OpDefCfaOffset.

needsWinCFI()

static bool needsWinCFI ( const MachineFunction *  MF )

static

Definition at line 1623 of file AArch64LoadStoreOptimizer.cpp.

References llvm::MachineFunction::getFunction(), llvm::TargetMachine::getMCAsmInfo(), llvm::MachineFunction::getTarget(), llvm::Function::needsUnwindTableEntry(), and llvm::MCAsmInfo::usesWindowsCFI().

Referenced by areCandidatesToMergeOrPair().

STATISTIC() [1/8]

STATISTIC	(	NumConstOffsetFolded	,
		"Number of const offset of index address folded"
	)

STATISTIC() [2/8]

STATISTIC	(	NumFailedAlignmentCheck	,
		"Number of load/store pair transformation " "not passed the alignment check"
	)

STATISTIC() [3/8]

STATISTIC	(	NumLoadsFromStoresPromoted	,
		"Number of loads from stores promoted"
	)

STATISTIC() [4/8]

STATISTIC	(	NumPairCreated	,
		"Number of load/store pair instructions generated"
	)

STATISTIC() [5/8]

STATISTIC	(	NumPostFolded	,
		"Number of post-index updates folded"
	)

STATISTIC() [6/8]

STATISTIC	(	NumPreFolded	,
		"Number of pre-index updates folded"
	)

STATISTIC() [7/8]

STATISTIC	(	NumUnscaledPairCreated	,
		"Number of load/store from unscaled generated"
	)

STATISTIC() [8/8]

STATISTIC	(	NumZeroStoresPromoted	,
		"Number of narrow zero stores promoted"
	)

tryToFindRegisterToRename()

static std::optional< MCPhysReg > tryToFindRegisterToRename ( const MachineFunction &  MF, Register  Reg, LiveRegUnits &  DefinedInBB, LiveRegUnits &  UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &  RequiredClasses, const TargetRegisterInfo *  TRI  )

static

Definition at line 1883 of file AArch64LoadStoreOptimizer.cpp.

References llvm::LiveRegUnits::addReg(), llvm::all_of(), llvm::any_of(), llvm::LiveRegUnits::available(), llvm::CallingConv::C, llvm::dbgs(), llvm::MachineFunction::getRegInfo(), LLVM_DEBUG, llvm::printReg(), and TRI.

Referenced by findRenameRegForSameLdStRegPair().

updateDefinedRegisters()

static void updateDefinedRegisters ( MachineInstr &  MI, LiveRegUnits &  Units, const TargetRegisterInfo *  TRI  )

static

Definition at line 954 of file AArch64LoadStoreOptimizer.cpp.

References llvm::LiveRegUnits::addReg(), MI, llvm::phys_regs_and_masks(), and llvm::LiveRegUnits::removeReg().

Variable Documentation

EnableRenaming

cl::opt< bool > EnableRenaming("aarch64-load-store-renaming", cl::init(true), cl::Hidden) ( "aarch64-load-store-renaming"  , cl::init(true)  , cl::Hidden    )

static

LdStConstLimit

cl::opt< unsigned > LdStConstLimit("aarch64-load-store-const-scan-limit", cl::init(10), cl::Hidden) ( "aarch64-load-store-const-scan-limit"  , cl::init(10)  , cl::Hidden    )

static

LdStLimit

cl::opt< unsigned > LdStLimit("aarch64-load-store-scan-limit", cl::init(20), cl::Hidden) ( "aarch64-load-store-scan-limit"  , cl::init(20)  , cl::Hidden    )

static

Referenced by canRenameUpToDef().

UpdateLimit

cl::opt< unsigned > UpdateLimit("aarch64-update-scan-limit", cl::init(100), cl::Hidden) ( "aarch64-update-scan-limit"  , cl::init(100)  , cl::Hidden    )

static