#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCDwarf.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include <cassert>
#include <cstdint>
#include <functional>
#include <iterator>
#include <limits>
#include <optional>

Macros
#define	DEBUG_TYPE "aarch64-ldst-opt"

#define	AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"

Functions
	STATISTIC (NumPairCreated, "Number of load/store pair instructions generated")

	STATISTIC (NumPostFolded, "Number of post-index updates folded")

	STATISTIC (NumPreFolded, "Number of pre-index updates folded")

	STATISTIC (NumUnscaledPairCreated, "Number of load/store from unscaled generated")

	STATISTIC (NumZeroStoresPromoted, "Number of narrow zero stores promoted")

	STATISTIC (NumLoadsFromStoresPromoted, "Number of loads from stores promoted")

	STATISTIC (NumFailedAlignmentCheck, "Number of load/store pair transformation " "not passed the alignment check")

	STATISTIC (NumConstOffsetFolded, "Number of const offset of index address folded")

	DEBUG_COUNTER (RegRenamingCounter, DEBUG_TYPE "-reg-renaming", "Controls which pairs are considered for renaming")

	INITIALIZE_PASS (AArch64LoadStoreOpt, "aarch64-ldst-opt", AARCH64_LOAD_STORE_OPT_NAME, false, false) static bool isNarrowStore(unsigned Opc)

static bool	isTagStore (const MachineInstr &MI)

static unsigned	getMatchingNonSExtOpcode (unsigned Opc, bool *IsValidLdStrOpc=nullptr)

static unsigned	getMatchingWideOpcode (unsigned Opc)

static unsigned	getMatchingPairOpcode (unsigned Opc)

static unsigned	isMatchingStore (MachineInstr &LoadInst, MachineInstr &StoreInst)

static unsigned	getPreIndexedOpcode (unsigned Opc)

static unsigned	getBaseAddressOpcode (unsigned Opc)

static unsigned	getPostIndexedOpcode (unsigned Opc)

static bool	isPreLdStPairCandidate (MachineInstr &FirstMI, MachineInstr &MI)

static void	getPrePostIndexedMemOpInfo (const MachineInstr &MI, int &Scale, int &MinOffset, int &MaxOffset)

static MachineOperand &	getLdStRegOp (MachineInstr &MI, unsigned PairedRegOp=0)

static bool	isLdOffsetInRangeOfSt (MachineInstr &LoadInst, MachineInstr &StoreInst, const AArch64InstrInfo *TII)

static bool	isPromotableZeroStoreInst (MachineInstr &MI)

static bool	isPromotableLoadFromStore (MachineInstr &MI)

static bool	isMergeableLdStUpdate (MachineInstr &MI, AArch64FunctionInfo &AFI)

static bool	isMergeableIndexLdSt (MachineInstr &MI, int &Scale)

static bool	isRewritableImplicitDef (unsigned Opc)

static bool	forAllMIsUntilDef (MachineInstr &MI, MCPhysReg DefReg, const TargetRegisterInfo *TRI, unsigned Limit, std::function< bool(MachineInstr &, bool)> &Fn)

static void	updateDefinedRegisters (MachineInstr &MI, LiveRegUnits &Units, const TargetRegisterInfo *TRI)

static bool	inBoundsForPair (bool IsUnscaled, int Offset, int OffsetStride)

static int	alignTo (int Num, int PowOf2)

static bool	mayAlias (MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)

static bool	needsWinCFI (const MachineFunction *MF)

static bool	areCandidatesToMergeOrPair (MachineInstr &FirstMI, MachineInstr &MI, LdStPairFlags &Flags, const AArch64InstrInfo *TII)

static bool	canRenameMOP (const MachineOperand &MOP, const TargetRegisterInfo *TRI)

static bool	canRenameUpToDef (MachineInstr &FirstMI, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)

static bool	canRenameUntilSecondLoad (MachineInstr &FirstLoad, MachineInstr &SecondLoad, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)

static std::optional< MCPhysReg >	tryToFindRegisterToRename (const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)

static std::optional< MCPhysReg >	findRenameRegForSameLdStRegPair (std::optional< bool > MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)

static MachineBasicBlock::iterator	maybeMoveCFI (MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI)

Variables
static cl::opt< unsigned >	LdStLimit ("aarch64-load-store-scan-limit", cl::init(20), cl::Hidden)

static cl::opt< unsigned >	UpdateLimit ("aarch64-update-scan-limit", cl::init(100), cl::Hidden)

static cl::opt< unsigned >	LdStConstLimit ("aarch64-load-store-const-scan-limit", cl::init(10), cl::Hidden)

static cl::opt< bool >	EnableRenaming ("aarch64-load-store-renaming", cl::init(true), cl::Hidden)

Macro Definition Documentation

◆ AARCH64_LOAD_STORE_OPT_NAME

#define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"

Definition at line 90 of file AArch64LoadStoreOptimizer.cpp.

◆ DEBUG_TYPE

#define DEBUG_TYPE "aarch64-ldst-opt"

Definition at line 55 of file AArch64LoadStoreOptimizer.cpp.

Function Documentation

◆ alignTo()

static int alignTo	(	int	Num,
		int	PowOf2
	)

static

Definition at line 1395 of file AArch64LoadStoreOptimizer.cpp.

◆ areCandidatesToMergeOrPair()

static bool areCandidatesToMergeOrPair	(	MachineInstr &	FirstMI,
		MachineInstr &	MI,
		LdStPairFlags &	Flags,
		const AArch64InstrInfo *	TII
	)

static

Definition at line 1481 of file AArch64LoadStoreOptimizer.cpp.

References assert(), llvm::MachineInstr::FrameDestroy, llvm::MachineInstr::FrameSetup, getMatchingNonSExtOpcode(), getMatchingPairOpcode(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::hasOrderedMemoryRef(), llvm::AArch64InstrInfo::isPreLdSt(), isPreLdStPairCandidate(), MI, needsWinCFI(), and TII.

◆ canRenameMOP()

static bool canRenameMOP	(	const MachineOperand &	MOP,
		const TargetRegisterInfo *	TRI
	)

static

Definition at line 1542 of file AArch64LoadStoreOptimizer.cpp.

References llvm::dbgs(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getParent(), llvm::MachineOperand::getReg(), llvm::MachineOperand::isDef(), llvm::MachineOperand::isEarlyClobber(), llvm::MachineOperand::isImplicit(), llvm::MachineOperand::isReg(), llvm::MachineOperand::isRenamable(), isRewritableImplicitDef(), llvm::MachineOperand::isTied(), LLVM_DEBUG, and TRI.

Referenced by canRenameUpToDef().

◆ canRenameUntilSecondLoad()

static bool canRenameUntilSecondLoad	(	MachineInstr &	FirstLoad,
		MachineInstr &	SecondLoad,
		LiveRegUnits &	UsedInBetween,
		SmallPtrSetImpl< const TargetRegisterClass * > &	RequiredClasses,
		const TargetRegisterInfo *	TRI
	)

static

Definition at line 1682 of file AArch64LoadStoreOptimizer.cpp.

References llvm::LiveRegUnits::accumulate(), for(), llvm::ilist_node_impl< OptionsT >::getIterator(), getLdStRegOp(), llvm::MachineOperand::getReg(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::MachineInstr::isPseudo(), MI, Success, and TRI.

Referenced by findRenameRegForSameLdStRegPair().

◆ canRenameUpToDef()

static bool canRenameUpToDef	(	MachineInstr &	FirstMI,
		LiveRegUnits &	UsedInBetween,
		SmallPtrSetImpl< const TargetRegisterClass * > &	RequiredClasses,
		const TargetRegisterInfo *	TRI
	)

static

◆ DEBUG_COUNTER()

DEBUG_COUNTER	(	RegRenamingCounter	,
		DEBUG_TYPE "-reg-renaming"	,
		"Controls which pairs are considered for renaming"
	)

◆ findRenameRegForSameLdStRegPair()

static std::optional< MCPhysReg > findRenameRegForSameLdStRegPair	(	std::optional< bool >	MaybeCanRename,
		MachineInstr &	FirstMI,
		MachineInstr &	MI,
		Register	Reg,
		LiveRegUnits &	DefinedInBB,
		LiveRegUnits &	UsedInBetween,
		SmallPtrSetImpl< const TargetRegisterClass * > &	RequiredClasses,
		const TargetRegisterInfo *	TRI
	)

static

Definition at line 1770 of file AArch64LoadStoreOptimizer.cpp.

References canRenameUntilSecondLoad(), canRenameUpToDef(), getLdStRegOp(), llvm::MachineBasicBlock::getParent(), llvm::MachineInstr::getParent(), getReg(), llvm::MachineFunction::getRegInfo(), llvm::MachineInstr::mayLoad(), MI, llvm::DebugCounter::shouldExecute(), llvm::MachineRegisterInfo::tracksLiveness(), TRI, and tryToFindRegisterToRename().

◆ forAllMIsUntilDef()

static bool forAllMIsUntilDef	(	MachineInstr &	MI,
		MCPhysReg	DefReg,
		const TargetRegisterInfo *	TRI,
		unsigned	Limit,
		std::function< bool(MachineInstr &, bool)> &	Fn
	)

static

Definition at line 932 of file AArch64LoadStoreOptimizer.cpp.

References llvm::any_of(), I, llvm::MachineBasicBlock::instr_rend(), llvm::instructionsWithoutDebug(), MBB, MI, and TRI.

Referenced by canRenameUpToDef().

◆ getBaseAddressOpcode()

static unsigned getBaseAddressOpcode ( unsigned Opc )

static

Definition at line 515 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable.

◆ getLdStRegOp()

static MachineOperand & getLdStRegOp	(	MachineInstr &	MI,
		unsigned	PairedRegOp = `0`
	)

static

Definition at line 682 of file AArch64LoadStoreOptimizer.cpp.

References assert(), Idx, llvm::AArch64InstrInfo::isPairedLdSt(), llvm::AArch64InstrInfo::isPreLdSt(), and MI.

Referenced by canRenameUntilSecondLoad(), canRenameUpToDef(), findRenameRegForSameLdStRegPair(), and isPromotableZeroStoreInst().

◆ getMatchingNonSExtOpcode()

static unsigned getMatchingNonSExtOpcode	(	unsigned	Opc,
		bool *	IsValidLdStrOpc = `nullptr`
	)

static

Definition at line 278 of file AArch64LoadStoreOptimizer.cpp.

Referenced by areCandidatesToMergeOrPair().

◆ getMatchingPairOpcode()

static unsigned getMatchingPairOpcode ( unsigned Opc )

static

Definition at line 350 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable.

Referenced by areCandidatesToMergeOrPair().

◆ getMatchingWideOpcode()

static unsigned getMatchingWideOpcode ( unsigned Opc )

static

Definition at line 331 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable.

◆ getPostIndexedOpcode()

static unsigned getPostIndexedOpcode ( unsigned Opc )

static

Definition at line 551 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable.

◆ getPreIndexedOpcode()

static unsigned getPreIndexedOpcode ( unsigned Opc )

static

Definition at line 442 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable.

◆ getPrePostIndexedMemOpInfo()

static void getPrePostIndexedMemOpInfo	(	const MachineInstr &	MI,
		int &	Scale,
		int &	MinOffset,
		int &	MaxOffset
	)

static

Definition at line 664 of file AArch64LoadStoreOptimizer.cpp.

References llvm::AArch64InstrInfo::getMemScale(), llvm::AArch64InstrInfo::isPairedLdSt(), isTagStore(), and MI.

◆ inBoundsForPair()

static bool inBoundsForPair	(	bool	IsUnscaled,
		int	Offset,
		int	OffsetStride
	)

static

Definition at line 1378 of file AArch64LoadStoreOptimizer.cpp.

References llvm::Offset.

◆ INITIALIZE_PASS()

INITIALIZE_PASS	(	AArch64LoadStoreOpt	,
		"aarch64-ldst-opt"	,
		AARCH64_LOAD_STORE_OPT_NAME	,
		false	,
		false
	)

Definition at line 249 of file AArch64LoadStoreOptimizer.cpp.

◆ isLdOffsetInRangeOfSt()

static bool isLdOffsetInRangeOfSt	(	MachineInstr &	LoadInst,
		MachineInstr &	StoreInst,
		const AArch64InstrInfo *	TII
	)

static

Definition at line 693 of file AArch64LoadStoreOptimizer.cpp.

References assert(), llvm::MachineOperand::getImm(), llvm::AArch64InstrInfo::getLdStOffsetOp(), isMatchingStore(), and TII.

◆ isMatchingStore()

static unsigned isMatchingStore	(	MachineInstr &	LoadInst,
		MachineInstr &	StoreInst
	)

static

Definition at line 412 of file AArch64LoadStoreOptimizer.cpp.

References llvm::Instruction::getOpcode(), and llvm_unreachable.

Referenced by isLdOffsetInRangeOfSt().

◆ isMergeableIndexLdSt()

static bool isMergeableIndexLdSt	(	MachineInstr &	MI,
		int &	Scale
	)

static

Definition at line 802 of file AArch64LoadStoreOptimizer.cpp.

References MI.

◆ isMergeableLdStUpdate()

static bool isMergeableLdStUpdate	(	MachineInstr &	MI,
		AArch64FunctionInfo &	AFI
	)

static

Definition at line 736 of file AArch64LoadStoreOptimizer.cpp.

References llvm::AArch64InstrInfo::getLdStBaseOp(), llvm::AArch64InstrInfo::getLdStOffsetOp(), llvm::MachineOperand::getReg(), isImm(), llvm::AArch64FunctionInfo::isMTETagged(), and MI.

◆ isPreLdStPairCandidate()

static bool isPreLdStPairCandidate	(	MachineInstr &	FirstMI,
		MachineInstr &	MI
	)

static

Definition at line 630 of file AArch64LoadStoreOptimizer.cpp.

References llvm::MachineInstr::getOpcode(), and MI.

Referenced by areCandidatesToMergeOrPair().

◆ isPromotableLoadFromStore()

static bool isPromotableLoadFromStore ( MachineInstr & MI )

static

Definition at line 718 of file AArch64LoadStoreOptimizer.cpp.

References MI.

◆ isPromotableZeroStoreInst()

static bool isPromotableZeroStoreInst ( MachineInstr & MI )

static

Definition at line 711 of file AArch64LoadStoreOptimizer.cpp.

References getLdStRegOp(), llvm::MachineOperand::getReg(), and MI.

◆ isRewritableImplicitDef()

static bool isRewritableImplicitDef ( unsigned Opc )

static

Definition at line 836 of file AArch64LoadStoreOptimizer.cpp.

Referenced by canRenameMOP().

◆ isTagStore()

static bool isTagStore ( const MachineInstr & MI )

static

Definition at line 266 of file AArch64LoadStoreOptimizer.cpp.

References MI.

Referenced by getPrePostIndexedMemOpInfo().

◆ mayAlias()

static bool mayAlias	(	MachineInstr &	MIa,
		SmallVectorImpl< MachineInstr * > &	MemInsns,
		AliasAnalysis *	AA
	)

static

Definition at line 1399 of file AArch64LoadStoreOptimizer.cpp.

References llvm::dbgs(), LLVM_DEBUG, and llvm::MachineInstr::mayAlias().

Referenced by llvm::HexagonInstrInfo::expandPostRAPseudo().

◆ maybeMoveCFI()

static MachineBasicBlock::iterator maybeMoveCFI	(	MachineInstr &	MI,
		MachineBasicBlock::iterator	MaybeCFI
	)

static

Definition at line 2076 of file AArch64LoadStoreOptimizer.cpp.

References assert(), End, llvm::MachineInstr::FrameDestroy, llvm::MachineInstr::FrameSetup, llvm::MachineFunction::getFrameInstructions(), llvm::MCCFIInstruction::getOperation(), MI, llvm::MCCFIInstruction::OpDefCfa, and llvm::MCCFIInstruction::OpDefCfaOffset.

◆ needsWinCFI()

static bool needsWinCFI ( const MachineFunction * MF )

static

Definition at line 1474 of file AArch64LoadStoreOptimizer.cpp.

References llvm::MachineFunction::getFunction(), llvm::TargetMachine::getMCAsmInfo(), llvm::MachineFunction::getTarget(), llvm::Function::needsUnwindTableEntry(), and llvm::MCAsmInfo::usesWindowsCFI().

Referenced by areCandidatesToMergeOrPair().

◆ STATISTIC() [1/8]

STATISTIC	(	NumConstOffsetFolded	,
		"Number of const offset of index address folded"
	)

◆ STATISTIC() [2/8]

STATISTIC	(	NumFailedAlignmentCheck	,
		"Number of load/store pair transformation " "not passed the alignment check"
	)

◆ STATISTIC() [3/8]

STATISTIC	(	NumLoadsFromStoresPromoted	,
		"Number of loads from stores promoted"
	)

◆ STATISTIC() [4/8]

STATISTIC	(	NumPairCreated	,
		"Number of load/store pair instructions generated"
	)

◆ STATISTIC() [5/8]

STATISTIC	(	NumPostFolded	,
		"Number of post-index updates folded"
	)

◆ STATISTIC() [6/8]

STATISTIC	(	NumPreFolded	,
		"Number of pre-index updates folded"
	)

◆ STATISTIC() [7/8]

STATISTIC	(	NumUnscaledPairCreated	,
		"Number of load/store from unscaled generated"
	)

◆ STATISTIC() [8/8]

STATISTIC	(	NumZeroStoresPromoted	,
		"Number of narrow zero stores promoted"
	)

◆ tryToFindRegisterToRename()

static std::optional< MCPhysReg > tryToFindRegisterToRename	(	const MachineFunction &	MF,
		Register	Reg,
		LiveRegUnits &	DefinedInBB,
		LiveRegUnits &	UsedInBetween,
		SmallPtrSetImpl< const TargetRegisterClass * > &	RequiredClasses,
		const TargetRegisterInfo *	TRI
	)

static

Definition at line 1726 of file AArch64LoadStoreOptimizer.cpp.

References llvm::LiveRegUnits::addReg(), llvm::all_of(), llvm::any_of(), llvm::LiveRegUnits::available(), llvm::CallingConv::C, llvm::dbgs(), llvm::MachineFunction::getRegInfo(), LLVM_DEBUG, llvm::printReg(), and TRI.

Referenced by findRenameRegForSameLdStRegPair().

◆ updateDefinedRegisters()

static void updateDefinedRegisters	(	MachineInstr &	MI,
		LiveRegUnits &	Units,
		const TargetRegisterInfo *	TRI
	)

static

Definition at line 954 of file AArch64LoadStoreOptimizer.cpp.

References llvm::LiveRegUnits::addReg(), MI, llvm::phys_regs_and_masks(), and llvm::LiveRegUnits::removeReg().

Variable Documentation

◆ EnableRenaming

cl::opt< bool > EnableRenaming("aarch64-load-store-renaming", cl::init(true), cl::Hidden)	(	"aarch64-load-store-renaming"	,
		cl::init(true)	,
		cl::Hidden
	)

static

◆ LdStConstLimit

cl::opt< unsigned > LdStConstLimit("aarch64-load-store-const-scan-limit", cl::init(10), cl::Hidden)	(	"aarch64-load-store-const-scan-limit"	,
		cl::init(10)	,
		cl::Hidden
	)

static

◆ LdStLimit

cl::opt< unsigned > LdStLimit("aarch64-load-store-scan-limit", cl::init(20), cl::Hidden)	(	"aarch64-load-store-scan-limit"	,
		cl::init(20)	,
		cl::Hidden
	)

static

Referenced by canRenameUpToDef().

◆ UpdateLimit

cl::opt< unsigned > UpdateLimit("aarch64-update-scan-limit", cl::init(100), cl::Hidden)	(	"aarch64-update-scan-limit"	,
		cl::init(100)	,
		cl::Hidden
	)

static

Macros

Functions

Variables

Macro Definition Documentation

◆ AARCH64_LOAD_STORE_OPT_NAME

◆ DEBUG_TYPE

Function Documentation

◆ alignTo()

◆ areCandidatesToMergeOrPair()

◆ canRenameMOP()

◆ canRenameUntilSecondLoad()

◆ canRenameUpToDef()

◆ DEBUG_COUNTER()

◆ findRenameRegForSameLdStRegPair()

◆ forAllMIsUntilDef()

◆ getBaseAddressOpcode()

◆ getLdStRegOp()

◆ getMatchingNonSExtOpcode()

◆ getMatchingPairOpcode()

◆ getMatchingWideOpcode()

◆ getPostIndexedOpcode()

◆ getPreIndexedOpcode()

◆ getPrePostIndexedMemOpInfo()

◆ inBoundsForPair()

◆ INITIALIZE_PASS()

◆ isLdOffsetInRangeOfSt()

◆ isMatchingStore()

◆ isMergeableIndexLdSt()

◆ isMergeableLdStUpdate()

◆ isPreLdStPairCandidate()

◆ isPromotableLoadFromStore()

◆ isPromotableZeroStoreInst()

◆ isRewritableImplicitDef()

◆ isTagStore()

◆ mayAlias()

◆ maybeMoveCFI()

◆ needsWinCFI()

◆ STATISTIC() [1/8]

◆ STATISTIC() [2/8]

◆ STATISTIC() [3/8]

◆ STATISTIC() [4/8]

◆ STATISTIC() [5/8]

◆ STATISTIC() [6/8]

◆ STATISTIC() [7/8]

◆ STATISTIC() [8/8]

◆ tryToFindRegisterToRename()

◆ updateDefinedRegisters()

Variable Documentation

◆ EnableRenaming

◆ LdStConstLimit

◆ LdStLimit

◆ UpdateLimit