#include "AArch64InstrInfo.h"
#include "AArch64MachineFunctionInfo.h"
#include "AArch64Subtarget.h"
#include "MCTargetDesc/AArch64AddressingModes.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCDwarf.h"
#include "llvm/Pass.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugCounter.h"
#include "llvm/Support/ErrorHandling.h"
#include <cassert>
#include <cstdint>
#include <functional>
#include <iterator>
#include <limits>
#include <optional>

Macros
#define	DEBUG_TYPE "aarch64-ldst-opt"
#define	AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"

Functions
	STATISTIC (NumPairCreated, "Number of load/store pair instructions generated")
	STATISTIC (NumPostFolded, "Number of post-index updates folded")
	STATISTIC (NumPreFolded, "Number of pre-index updates folded")
	STATISTIC (NumUnscaledPairCreated, "Number of load/store from unscaled generated")
	STATISTIC (NumZeroStoresPromoted, "Number of narrow zero stores promoted")
	STATISTIC (NumLoadsFromStoresPromoted, "Number of loads from stores promoted")
	STATISTIC (NumFailedAlignmentCheck, "Number of load/store pair transformation " "not passed the alignment check")
	STATISTIC (NumConstOffsetFolded, "Number of const offset of index address folded")
	STATISTIC (NumUMOVFoldedToFPRStore, "Number of UMOV + GPR stores folded to FPR stores")
	DEBUG_COUNTER (RegRenamingCounter, DEBUG_TYPE "-reg-renaming", "Controls which pairs are considered for renaming")
	INITIALIZE_PASS (AArch64LoadStoreOptLegacy, "aarch64-ldst-opt", AARCH64_LOAD_STORE_OPT_NAME, false, false) static bool isNarrowStore(unsigned Opc)
static bool	isTagStore (const MachineInstr &MI)
static unsigned	getMatchingNonSExtOpcode (unsigned Opc, bool *IsValidLdStrOpc=nullptr)
static unsigned	getMatchingWideOpcode (unsigned Opc)
static unsigned	getMatchingPairOpcode (unsigned Opc)
static unsigned	isMatchingStore (MachineInstr &LoadInst, MachineInstr &StoreInst)
static unsigned	getPreIndexedOpcode (unsigned Opc)
static unsigned	getBaseAddressOpcode (unsigned Opc)
static unsigned	getPostIndexedOpcode (unsigned Opc)
static bool	isPreLdStPairCandidate (MachineInstr &FirstMI, MachineInstr &MI)
static void	getPrePostIndexedMemOpInfo (const MachineInstr &MI, int &Scale, int &MinOffset, int &MaxOffset)
static MachineOperand &	getLdStRegOp (MachineInstr &MI, unsigned PairedRegOp=0)
static bool	isLdOffsetInRangeOfSt (MachineInstr &LoadInst, MachineInstr &StoreInst, const AArch64InstrInfo *TII)
static bool	isPromotableZeroStoreInst (MachineInstr &MI)
static bool	isPromotableLoadFromStore (MachineInstr &MI)
static bool	isMergeableLdStUpdate (MachineInstr &MI, AArch64FunctionInfo &AFI)
static bool	isMergeableIndexLdSt (MachineInstr &MI, int &Scale)
static bool	isRewritableImplicitDef (const MachineInstr &MI, const MachineOperand &MO)
static bool	forAllMIsUntilDef (MachineInstr &MI, MCPhysReg DefReg, const TargetRegisterInfo *TRI, unsigned Limit, std::function< bool(MachineInstr &, bool)> &Fn)
static void	updateDefinedRegisters (MachineInstr &MI, LiveRegUnits &Units, const TargetRegisterInfo *TRI)
static void	addDebugSubstitutionsToTable (MachineFunction *MF, unsigned InstrNumToSet, MachineInstr &OriginalInstr, MachineInstr &MergedInstr)
	This function will add a new entry into the debugValueSubstitutions table when two instruction have been merged into a new one represented by `MergedInstr`.
static bool	inBoundsForPair (bool IsUnscaled, int Offset, int OffsetStride)
static int	alignTo (int Num, int PowOf2)
static bool	mayAlias (MachineInstr &MIa, SmallVectorImpl< MachineInstr * > &MemInsns, AliasAnalysis *AA)
static bool	areCandidatesToMergeOrPair (MachineInstr &FirstMI, MachineInstr &MI, LdStPairFlags &Flags, const AArch64InstrInfo *TII)
static bool	canRenameMOP (const MachineInstr &MI, const MachineOperand &MOP, const TargetRegisterInfo *TRI)
static bool	canRenameUpToDef (MachineInstr &FirstMI, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static bool	canRenameUntilSecondLoad (MachineInstr &FirstLoad, MachineInstr &SecondLoad, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static std::optional< MCPhysReg >	tryToFindRegisterToRename (const MachineFunction &MF, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static std::optional< MCPhysReg >	findRenameRegForSameLdStRegPair (std::optional< bool > MaybeCanRename, MachineInstr &FirstMI, MachineInstr &MI, Register Reg, LiveRegUnits &DefinedInBB, LiveRegUnits &UsedInBetween, SmallPtrSetImpl< const TargetRegisterClass * > &RequiredClasses, const TargetRegisterInfo *TRI)
static MachineBasicBlock::iterator	maybeMoveCFI (MachineInstr &MI, MachineBasicBlock::iterator MaybeCFI)
static unsigned	getGPRToFPRStoreOpcode (unsigned GPRStoreOpc)
static unsigned	getUMOVSubRegIdx (unsigned UMOVOpc)

Macro Definition Documentation

◆ AARCH64_LOAD_STORE_OPT_NAME

#define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass"

Definition at line 73 of file AArch64LoadStoreOptimizer.cpp.

Referenced by INITIALIZE_PASS().

◆ DEBUG_TYPE

#define DEBUG_TYPE "aarch64-ldst-opt"

Definition at line 54 of file AArch64LoadStoreOptimizer.cpp.

Function Documentation

◆ addDebugSubstitutionsToTable()

void addDebugSubstitutionsToTable	(	MachineFunction *	MF,
		unsigned	InstrNumToSet,
		MachineInstr &	OriginalInstr,
		MachineInstr &	MergedInstr )

static

This function will add a new entry into the debugValueSubstitutions table when two instruction have been merged into a new one represented by MergedInstr.

Definition at line 981 of file AArch64LoadStoreOptimizer.cpp.

References llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::MachineFunction::makeDebugValueSubstitution(), llvm::MachineInstr::operands(), llvm::MachineInstr::peekDebugInstrNum(), and Reg.

◆ alignTo()

int alignTo	(	int	Num,
		int	PowOf2 )

static

Definition at line 1575 of file AArch64LoadStoreOptimizer.cpp.

Referenced by llvm::TargetFrameLowering::alignSPAdjust().

◆ areCandidatesToMergeOrPair()

bool areCandidatesToMergeOrPair	(	MachineInstr &	FirstMI,
		MachineInstr &	MI,
		LdStPairFlags &	Flags,
		const AArch64InstrInfo *	TII )

static

Definition at line 1656 of file AArch64LoadStoreOptimizer.cpp.

References assert(), llvm::MachineInstr::FrameDestroy, llvm::MachineInstr::FrameSetup, getLdStRegOp(), getMatchingNonSExtOpcode(), getMatchingPairOpcode(), llvm::MachineInstr::getOpcode(), llvm::MachineOperand::getReg(), llvm::MachineInstr::hasOrderedMemoryRef(), llvm::AArch64InstrInfo::isPreLdSt(), isPreLdStPairCandidate(), MI, and TII.

◆ canRenameMOP()

bool canRenameMOP	(	const MachineInstr &	MI,
		const MachineOperand &	MOP,
		const TargetRegisterInfo *	TRI )

static

Definition at line 1726 of file AArch64LoadStoreOptimizer.cpp.

References llvm::dbgs(), llvm::MachineOperand::getReg(), llvm::MachineOperand::isDef(), llvm::MachineOperand::isEarlyClobber(), llvm::MachineOperand::isImplicit(), llvm::MachineOperand::isReg(), llvm::MachineOperand::isRenamable(), isRewritableImplicitDef(), llvm::MachineOperand::isTied(), LLVM_DEBUG, MI, and TRI.

Referenced by canRenameUpToDef().

◆ canRenameUntilSecondLoad()

bool canRenameUntilSecondLoad	(	MachineInstr &	FirstLoad,
		MachineInstr &	SecondLoad,
		LiveRegUnits &	UsedInBetween,
		SmallPtrSetImpl< const TargetRegisterClass * > &	RequiredClasses,
		const TargetRegisterInfo *	TRI )

static

Definition at line 1869 of file AArch64LoadStoreOptimizer.cpp.

References llvm::LiveRegUnits::accumulate(), for(), llvm::ilist_node_impl< OptionsT >::getIterator(), getLdStRegOp(), llvm::MachineOperand::getReg(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::MachineInstr::isPseudo(), MI, Success, and TRI.

Referenced by findRenameRegForSameLdStRegPair().

◆ canRenameUpToDef()

bool canRenameUpToDef	(	MachineInstr &	FirstMI,
		LiveRegUnits &	UsedInBetween,
		SmallPtrSetImpl< const TargetRegisterClass * > &	RequiredClasses,
		const TargetRegisterInfo *	TRI )

static

◆ DEBUG_COUNTER()

DEBUG_COUNTER	(	RegRenamingCounter	,
		DEBUG_TYPE "-reg-renaming"	,
		"Controls which pairs are considered for renaming"	)

References DEBUG_TYPE.

◆ findRenameRegForSameLdStRegPair()

std::optional< MCPhysReg > findRenameRegForSameLdStRegPair	(	std::optional< bool >	MaybeCanRename,
		MachineInstr &	FirstMI,
		MachineInstr &	MI,
		Register	Reg,
		LiveRegUnits &	DefinedInBB,
		LiveRegUnits &	UsedInBetween,
		SmallPtrSetImpl< const TargetRegisterClass * > &	RequiredClasses,
		const TargetRegisterInfo *	TRI )

static

Definition at line 1957 of file AArch64LoadStoreOptimizer.cpp.

References canRenameUntilSecondLoad(), canRenameUpToDef(), getLdStRegOp(), llvm::MachineBasicBlock::getParent(), llvm::MachineInstr::getParent(), getReg(), llvm::MachineFunction::getRegInfo(), llvm::MachineInstr::mayLoad(), MI, Reg, llvm::DebugCounter::shouldExecute(), llvm::MachineRegisterInfo::tracksLiveness(), TRI, and tryToFindRegisterToRename().

◆ forAllMIsUntilDef()

bool forAllMIsUntilDef	(	MachineInstr &	MI,
		MCPhysReg	DefReg,
		const TargetRegisterInfo *	TRI,
		unsigned	Limit,
		std::function< bool(MachineInstr &, bool)> &	Fn )

static

Definition at line 944 of file AArch64LoadStoreOptimizer.cpp.

References llvm::any_of(), I, llvm::instructionsWithoutDebug(), MBB, MI, and TRI.

Referenced by canRenameUpToDef().

◆ getBaseAddressOpcode()

unsigned getBaseAddressOpcode ( unsigned Opc )

static

Definition at line 515 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable, and Opc.

◆ getGPRToFPRStoreOpcode()

unsigned getGPRToFPRStoreOpcode ( unsigned GPRStoreOpc )

static

Definition at line 3015 of file AArch64LoadStoreOptimizer.cpp.

◆ getLdStRegOp()

MachineOperand & getLdStRegOp	(	MachineInstr &	MI,
		unsigned	PairedRegOp = 0 )

static

Definition at line 690 of file AArch64LoadStoreOptimizer.cpp.

References assert(), llvm::AArch64InstrInfo::isPairedLdSt(), llvm::AArch64InstrInfo::isPreLdSt(), and MI.

Referenced by areCandidatesToMergeOrPair(), canRenameUntilSecondLoad(), canRenameUpToDef(), findRenameRegForSameLdStRegPair(), and isPromotableZeroStoreInst().

◆ getMatchingNonSExtOpcode()

unsigned getMatchingNonSExtOpcode	(	unsigned	Opc,
		bool *	IsValidLdStrOpc = nullptr )

static

Definition at line 266 of file AArch64LoadStoreOptimizer.cpp.

References Opc.

Referenced by areCandidatesToMergeOrPair().

◆ getMatchingPairOpcode()

unsigned getMatchingPairOpcode ( unsigned Opc )

static

Definition at line 340 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable, and Opc.

Referenced by areCandidatesToMergeOrPair().

◆ getMatchingWideOpcode()

unsigned getMatchingWideOpcode ( unsigned Opc )

static

Definition at line 321 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable, and Opc.

◆ getPostIndexedOpcode()

unsigned getPostIndexedOpcode ( unsigned Opc )

static

Definition at line 551 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable, and Opc.

◆ getPreIndexedOpcode()

unsigned getPreIndexedOpcode ( unsigned Opc )

static

Definition at line 434 of file AArch64LoadStoreOptimizer.cpp.

References llvm_unreachable, and Opc.

◆ getPrePostIndexedMemOpInfo()

void getPrePostIndexedMemOpInfo	(	const MachineInstr &	MI,
		int &	Scale,
		int &	MinOffset,
		int &	MaxOffset )

static

Definition at line 672 of file AArch64LoadStoreOptimizer.cpp.

References llvm::AArch64InstrInfo::getMemScale(), llvm::AArch64InstrInfo::isPairedLdSt(), isTagStore(), and MI.

◆ getUMOVSubRegIdx()

unsigned getUMOVSubRegIdx ( unsigned UMOVOpc )

static

Definition at line 3059 of file AArch64LoadStoreOptimizer.cpp.

◆ inBoundsForPair()

bool inBoundsForPair	(	bool	IsUnscaled,
		int	Offset,
		int	OffsetStride )

static

Definition at line 1558 of file AArch64LoadStoreOptimizer.cpp.

References llvm::Offset.

◆ INITIALIZE_PASS()

INITIALIZE_PASS	(	AArch64LoadStoreOptLegacy	,
		"aarch64-ldst-opt"	,
		AARCH64_LOAD_STORE_OPT_NAME	,
		false	,
		false	)

Definition at line 237 of file AArch64LoadStoreOptimizer.cpp.

References AARCH64_LOAD_STORE_OPT_NAME, and Opc.

◆ isLdOffsetInRangeOfSt()

bool isLdOffsetInRangeOfSt	(	MachineInstr &	LoadInst,
		MachineInstr &	StoreInst,
		const AArch64InstrInfo *	TII )

static

Definition at line 701 of file AArch64LoadStoreOptimizer.cpp.

References assert(), llvm::MachineOperand::getImm(), llvm::AArch64InstrInfo::getLdStOffsetOp(), isMatchingStore(), and TII.

◆ isMatchingStore()

unsigned isMatchingStore	(	MachineInstr &	LoadInst,
		MachineInstr &	StoreInst )

static

Definition at line 404 of file AArch64LoadStoreOptimizer.cpp.

References llvm::Instruction::getOpcode(), and llvm_unreachable.

Referenced by isLdOffsetInRangeOfSt().

◆ isMergeableIndexLdSt()

bool isMergeableIndexLdSt	(	MachineInstr &	MI,
		int &	Scale )

static

Definition at line 814 of file AArch64LoadStoreOptimizer.cpp.

References MI, and Opc.

◆ isMergeableLdStUpdate()

bool isMergeableLdStUpdate	(	MachineInstr &	MI,
		AArch64FunctionInfo &	AFI )

static

Definition at line 744 of file AArch64LoadStoreOptimizer.cpp.

References llvm::AArch64InstrInfo::getLdStBaseOp(), llvm::AArch64InstrInfo::getLdStOffsetOp(), llvm::AArch64FunctionInfo::isMTETagged(), MI, and Opc.

◆ isPreLdStPairCandidate()

bool isPreLdStPairCandidate	(	MachineInstr &	FirstMI,
		MachineInstr &	MI )

static

Definition at line 638 of file AArch64LoadStoreOptimizer.cpp.

References llvm::MachineInstr::getOpcode(), and MI.

Referenced by areCandidatesToMergeOrPair().

◆ isPromotableLoadFromStore()

bool isPromotableLoadFromStore ( MachineInstr & MI )

static

Definition at line 726 of file AArch64LoadStoreOptimizer.cpp.

References MI.

◆ isPromotableZeroStoreInst()

bool isPromotableZeroStoreInst ( MachineInstr & MI )

static

Definition at line 719 of file AArch64LoadStoreOptimizer.cpp.

References getLdStRegOp(), llvm::MachineOperand::getReg(), MI, and Opc.

◆ isRewritableImplicitDef()

bool isRewritableImplicitDef	(	const MachineInstr &	MI,
		const MachineOperand &	MO )

static

Definition at line 848 of file AArch64LoadStoreOptimizer.cpp.

References llvm::MachineOperand::isRenamable(), and MI.

Referenced by canRenameMOP().

◆ isTagStore()

bool isTagStore ( const MachineInstr & MI )

static

Definition at line 254 of file AArch64LoadStoreOptimizer.cpp.

References MI.

Referenced by getPrePostIndexedMemOpInfo().

◆ mayAlias()

bool mayAlias	(	MachineInstr &	MIa,
		SmallVectorImpl< MachineInstr * > &	MemInsns,
		AliasAnalysis *	AA )

static

Definition at line 1579 of file AArch64LoadStoreOptimizer.cpp.

References llvm::dbgs(), LLVM_DEBUG, and llvm::MachineInstr::mayAlias().

Referenced by llvm::HexagonInstrInfo::expandPostRAPseudo().

◆ maybeMoveCFI()

MachineBasicBlock::iterator maybeMoveCFI	(	MachineInstr &	MI,
		MachineBasicBlock::iterator	MaybeCFI )

static

Definition at line 2259 of file AArch64LoadStoreOptimizer.cpp.

References assert(), llvm::End, llvm::MachineInstr::FrameDestroy, llvm::MachineInstr::FrameSetup, llvm::MachineFunction::getFrameInstructions(), llvm::MCCFIInstruction::getOperation(), MI, llvm::MCCFIInstruction::OpDefCfa, and llvm::MCCFIInstruction::OpDefCfaOffset.

◆ STATISTIC() [1/9]

STATISTIC	(	NumConstOffsetFolded	,
		"Number of const offset of index address folded"	)

◆ STATISTIC() [2/9]

STATISTIC	(	NumFailedAlignmentCheck	,
		"Number of load/store pair transformation " "not passed the alignment check"	)

◆ STATISTIC() [3/9]

STATISTIC	(	NumLoadsFromStoresPromoted	,
		"Number of loads from stores promoted"	)

◆ STATISTIC() [4/9]

STATISTIC	(	NumPairCreated	,
		"Number of load/store pair instructions generated"	)

◆ STATISTIC() [5/9]

STATISTIC	(	NumPostFolded	,
		"Number of post-index updates folded"	)

◆ STATISTIC() [6/9]

STATISTIC	(	NumPreFolded	,
		"Number of pre-index updates folded"	)

◆ STATISTIC() [7/9]

STATISTIC	(	NumUMOVFoldedToFPRStore	,
		"Number of UMOV + GPR stores folded to FPR stores"	)

◆ STATISTIC() [8/9]

STATISTIC	(	NumUnscaledPairCreated	,
		"Number of load/store from unscaled generated"	)

◆ STATISTIC() [9/9]

STATISTIC	(	NumZeroStoresPromoted	,
		"Number of narrow zero stores promoted"	)

◆ tryToFindRegisterToRename()

std::optional< MCPhysReg > tryToFindRegisterToRename	(	const MachineFunction &	MF,
		Register	Reg,
		LiveRegUnits &	DefinedInBB,
		LiveRegUnits &	UsedInBetween,
		SmallPtrSetImpl< const TargetRegisterClass * > &	RequiredClasses,
		const TargetRegisterInfo *	TRI )

static

Definition at line 1913 of file AArch64LoadStoreOptimizer.cpp.

References llvm::LiveRegUnits::addReg(), llvm::all_of(), llvm::any_of(), llvm::LiveRegUnits::available(), C(), llvm::dbgs(), llvm::MachineFunction::getRegInfo(), LLVM_DEBUG, llvm::printReg(), Reg, and TRI.

Referenced by findRenameRegForSameLdStRegPair().

◆ updateDefinedRegisters()

void updateDefinedRegisters	(	MachineInstr &	MI,
		LiveRegUnits &	Units,
		const TargetRegisterInfo *	TRI )

static

Definition at line 966 of file AArch64LoadStoreOptimizer.cpp.

References llvm::LiveRegUnits::addReg(), MI, llvm::phys_regs_and_masks(), llvm::LiveRegUnits::removeReg(), and TRI.

Macros

Functions

Macro Definition Documentation

◆ AARCH64_LOAD_STORE_OPT_NAME

◆ DEBUG_TYPE

Function Documentation

◆ addDebugSubstitutionsToTable()

◆ alignTo()

◆ areCandidatesToMergeOrPair()

◆ canRenameMOP()

◆ canRenameUntilSecondLoad()

◆ canRenameUpToDef()

◆ DEBUG_COUNTER()

◆ findRenameRegForSameLdStRegPair()

◆ forAllMIsUntilDef()

◆ getBaseAddressOpcode()

◆ getGPRToFPRStoreOpcode()

◆ getLdStRegOp()

◆ getMatchingNonSExtOpcode()

◆ getMatchingPairOpcode()

◆ getMatchingWideOpcode()

◆ getPostIndexedOpcode()

◆ getPreIndexedOpcode()

◆ getPrePostIndexedMemOpInfo()

◆ getUMOVSubRegIdx()

◆ inBoundsForPair()

◆ INITIALIZE_PASS()

◆ isLdOffsetInRangeOfSt()

◆ isMatchingStore()

◆ isMergeableIndexLdSt()

◆ isMergeableLdStUpdate()

◆ isPreLdStPairCandidate()

◆ isPromotableLoadFromStore()

◆ isPromotableZeroStoreInst()

◆ isRewritableImplicitDef()

◆ isTagStore()

◆ mayAlias()

◆ maybeMoveCFI()

◆ STATISTIC() [1/9]

◆ STATISTIC() [2/9]

◆ STATISTIC() [3/9]

◆ STATISTIC() [4/9]

◆ STATISTIC() [5/9]

◆ STATISTIC() [6/9]

◆ STATISTIC() [7/9]

◆ STATISTIC() [8/9]

◆ STATISTIC() [9/9]

◆ tryToFindRegisterToRename()

◆ updateDefinedRegisters()