This file defines the pass which inserts x86 AVX vzeroupper instructions before calls to SSE encoded functions. More...

#include "X86.h"
#include "X86InstrInfo.h"
#include "X86Subtarget.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineOperand.h"
#include "llvm/CodeGen/MachinePassManager.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/Analysis.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/Function.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include <cassert>

Macros
#define	DEBUG_TYPE "x86-insert-vzeroupper"

Functions
	STATISTIC (NumVZU, "Number of vzeroupper instructions inserted")
static const char *	getBlockExitStateName (BlockExitState ST)
static bool	isYmmOrZmmReg (MCRegister Reg)
	VZEROUPPER cleans state that is related to Y/ZMM0-15 only.
static bool	checkFnHasLiveInYmmOrZmm (MachineRegisterInfo &MRI)
static bool	clobbersAllYmmAndZmmRegs (const MachineOperand &MO)
static bool	hasYmmOrZmmReg (MachineInstr &MI)
static bool	callHasRegMask (MachineInstr &MI)
	Check if given call instruction has a RegMask operand.
static bool	insertVZeroUpper (MachineBasicBlock::iterator I, MachineBasicBlock &MBB, const TargetInstrInfo *TII)
	Insert a vzeroupper instruction before I.
static void	addDirtySuccessor (MachineBasicBlock &MBB, BlockStateMap &BlockStates, DirtySuccessorsWorkList &DirtySuccessors)
	Add MBB to the DirtySuccessors list if it hasn't already been added.
static bool	processBasicBlock (MachineBasicBlock &MBB, BlockStateMap &BlockStates, DirtySuccessorsWorkList &DirtySuccessors, bool IsX86INTR, const TargetInstrInfo *TII)
	Loop over all of the instructions in the basic block, inserting vzeroupper instructions before function calls.
static bool	insertVZeroUpper (MachineFunction &MF)
	Loop over all of the basic blocks, inserting vzeroupper instructions before function calls.

Variables
static cl::opt< bool >	UseVZeroUpper ("x86-use-vzeroupper", cl::Hidden, cl::desc("Minimize AVX to SSE transition penalty"), cl::init(true))

Detailed Description

This file defines the pass which inserts x86 AVX vzeroupper instructions before calls to SSE encoded functions.

This avoids transition latency penalty when transferring control between AVX encoded instructions and old SSE encoding mode.

Definition in file X86InsertVZeroUpper.cpp.

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "x86-insert-vzeroupper"

Definition at line 43 of file X86InsertVZeroUpper.cpp.

Function Documentation

◆ addDirtySuccessor()

void addDirtySuccessor	(	MachineBasicBlock &	MBB,
		BlockStateMap &	BlockStates,
		DirtySuccessorsWorkList &	DirtySuccessors )

static

Add MBB to the DirtySuccessors list if it hasn't already been added.

Definition at line 181 of file X86InsertVZeroUpper.cpp.

References MBB.

Referenced by insertVZeroUpper(), and processBasicBlock().

◆ callHasRegMask()

bool callHasRegMask ( MachineInstr & MI )

static

Check if given call instruction has a RegMask operand.

Definition at line 162 of file X86InsertVZeroUpper.cpp.

References assert(), and MI.

Referenced by processBasicBlock().

◆ checkFnHasLiveInYmmOrZmm()

bool checkFnHasLiveInYmmOrZmm ( MachineRegisterInfo & MRI )

static

Definition at line 127 of file X86InsertVZeroUpper.cpp.

References isYmmOrZmmReg(), and llvm::MachineRegisterInfo::liveins().

Referenced by insertVZeroUpper().

◆ clobbersAllYmmAndZmmRegs()

bool clobbersAllYmmAndZmmRegs ( const MachineOperand & MO )

static

Definition at line 135 of file X86InsertVZeroUpper.cpp.

References llvm::MachineOperand::clobbersPhysReg().

Referenced by hasYmmOrZmmReg().

◆ getBlockExitStateName()

const char * getBlockExitStateName ( BlockExitState ST )

static

Definition at line 107 of file X86InsertVZeroUpper.cpp.

References llvm_unreachable.

Referenced by processBasicBlock().

◆ hasYmmOrZmmReg()

bool hasYmmOrZmmReg ( MachineInstr & MI )

static

Definition at line 147 of file X86InsertVZeroUpper.cpp.

References clobbersAllYmmAndZmmRegs(), isYmmOrZmmReg(), and MI.

Referenced by processBasicBlock().

◆ insertVZeroUpper() [1/2]

bool insertVZeroUpper	(	MachineBasicBlock::iterator	I,
		MachineBasicBlock &	MBB,
		const TargetInstrInfo *	TII )

static

Insert a vzeroupper instruction before I.

Definition at line 172 of file X86InsertVZeroUpper.cpp.

References llvm::BuildMI(), I, MBB, and TII.

Referenced by insertVZeroUpper(), processBasicBlock(), and llvm::X86InsertVZeroUpperPass::run().

◆ insertVZeroUpper() [2/2]

bool insertVZeroUpper ( MachineFunction & MF )

static

Loop over all of the basic blocks, inserting vzeroupper instructions before function calls.

Definition at line 279 of file X86InsertVZeroUpper.cpp.

References addDirtySuccessor(), assert(), checkFnHasLiveInYmmOrZmm(), llvm::dbgs(), llvm::MachineFunction::front(), llvm::Function::getCallingConv(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getNumBlockIDs(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), insertVZeroUpper(), LLVM_DEBUG, MBB, processBasicBlock(), llvm::MachineRegisterInfo::reg_nodbg_empty(), TII, UseVZeroUpper, and llvm::CallingConv::X86_INTR.

◆ isYmmOrZmmReg()

bool isYmmOrZmmReg ( MCRegister Reg )

static

VZEROUPPER cleans state that is related to Y/ZMM0-15 only.

Thus, there is no need to check for Y/ZMM16 and above.

Definition at line 122 of file X86InsertVZeroUpper.cpp.

References Reg.

Referenced by checkFnHasLiveInYmmOrZmm(), and hasYmmOrZmmReg().

◆ processBasicBlock()

bool processBasicBlock	(	MachineBasicBlock &	MBB,
		BlockStateMap &	BlockStates,
		DirtySuccessorsWorkList &	DirtySuccessors,
		bool	IsX86INTR,
		const TargetInstrInfo *	TII )

static

Loop over all of the instructions in the basic block, inserting vzeroupper instructions before function calls.

Definition at line 192 of file X86InsertVZeroUpper.cpp.

References addDirtySuccessor(), callHasRegMask(), llvm::dbgs(), getBlockExitStateName(), hasYmmOrZmmReg(), insertVZeroUpper(), LLVM_DEBUG, MBB, MI, and TII.

Referenced by insertVZeroUpper().

◆ STATISTIC()

STATISTIC	(	NumVZU	,
		"Number of vzeroupper instructions inserted"	)

Variable Documentation

◆ UseVZeroUpper

cl::opt< bool > UseVZeroUpper("x86-use-vzeroupper", cl::Hidden, cl::desc("Minimize AVX to SSE transition penalty"), cl::init(true))	(	"x86-use-vzeroupper"	,
		cl::Hidden	,
		cl::desc("Minimize AVX to SSE transition penalty")	,
		cl::init(true)	)

static

Referenced by insertVZeroUpper().

Macros

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ DEBUG_TYPE

Function Documentation

◆ addDirtySuccessor()

◆ callHasRegMask()

◆ checkFnHasLiveInYmmOrZmm()

◆ clobbersAllYmmAndZmmRegs()

◆ getBlockExitStateName()

◆ hasYmmOrZmmReg()

◆ insertVZeroUpper() [1/2]

◆ insertVZeroUpper() [2/2]

◆ isYmmOrZmmReg()

◆ processBasicBlock()

◆ STATISTIC()

Variable Documentation

◆ UseVZeroUpper