#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
#include "llvm/CodeGen/GlobalISel/CallLowering.h"
#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetFrameLowering.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/Instructions.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include <numeric>
#include <optional>

Include dependency graph for LegalizerHelper.cpp:

Macros
#define	DEBUG_TYPE "legalizer"

#define	RTLIBCASE_INT(LibcallPrefix)

#define	RTLIBCASE(LibcallPrefix)

Functions
static std::pair< int, int >	getNarrowTypeBreakDown (LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy)
	Try to break down `OrigTy` into `NarrowTy` sized pieces.

static Type *	getFloatTypeForLLT (LLVMContext &Ctx, LLT Ty)

static void	getUnmergeResults (SmallVectorImpl< Register > &Regs, const MachineInstr &MI)
	Append the result registers of G_UNMERGE_VALUES `MI` to `Regs`.

static RTLIB::Libcall	getRTLibDesc (unsigned Opcode, unsigned Size)

static bool	isLibCallInTailPosition (MachineInstr &MI, const TargetInstrInfo &TII, MachineRegisterInfo &MRI)
	True if an instruction is in tail position in its caller.

static LegalizerHelper::LegalizeResult	simpleLibcall (MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType)

static RTLIB::Libcall	getConvRTLibDesc (unsigned Opcode, Type ToType, Type FromType)

static LegalizerHelper::LegalizeResult	conversionLibcall (MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type ToType, Type FromType)

static void	getUnmergePieces (SmallVectorImpl< Register > &Pieces, MachineIRBuilder &B, Register Src, LLT Ty)

static Register	getBitcastWiderVectorElementOffset (MachineIRBuilder &B, Register Idx, unsigned NewEltSize, unsigned OldEltSize)
	Figure out the bit offset into a register when coercing a vector index for the wide element type.

static Register	buildBitFieldInsert (MachineIRBuilder &B, Register TargetReg, Register InsertReg, Register OffsetBits)
	Emit code to insert `InsertReg` into `TargetRet` at `OffsetBits` in `TargetReg`, while preserving other bits in `TargetReg`.

static Register	clampDynamicVectorIndex (MachineIRBuilder &B, Register IdxReg, LLT VecTy)

static bool	hasSameNumEltsOnAllVectorOperands (GenericMachineInstr &MI, MachineRegisterInfo &MRI, std::initializer_list< unsigned > NonVecOpIndices)
	Check that all vector operands have same number of elements.

static void	makeDstOps (SmallVectorImpl< DstOp > &DstOps, LLT Ty, unsigned NumElts)
	Fill `DstOps` with DstOps that have same number of elements combined as the Ty.

static void	broadcastSrcOp (SmallVectorImpl< SrcOp > &Ops, unsigned N, MachineOperand &Op)
	Operand `Op` is used on `N` sub-instructions.

static unsigned	getScalarOpcForReduction (unsigned Opc)

static bool	isNonZeroModBitWidthOrUndef (const MachineRegisterInfo &MRI, Register Reg, unsigned BW)

static CmpInst::Predicate	minMaxToCompare (unsigned Opc)

static MachineInstrBuilder	SwapN (unsigned N, DstOp Dst, MachineIRBuilder &B, MachineInstrBuilder Src, APInt Mask)

static bool	shouldLowerMemFuncForSize (const MachineFunction &MF)

static bool	findGISelOptimalMemOpLowering (std::vector< LLT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, const TargetLowering &TLI)

static Type *	getTypeForLLT (LLT Ty, LLVMContext &C)

static Register	getMemsetValue (Register Val, LLT Ty, MachineIRBuilder &MIB)

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "legalizer"

Definition at line 41 of file LegalizerHelper.cpp.

◆ RTLIBCASE

#define RTLIBCASE ( LibcallPrefix )

Value:

  do {                                                                         \
    switch (Size) {                                                            \
    case 32:                                                                   \
      return RTLIB::LibcallPrefix##32;                                         \
    case 64:                                                                   \
      return RTLIB::LibcallPrefix##64;                                         \
    case 80:                                                                   \
      return RTLIB::LibcallPrefix##80;                                         \
    case 128:                                                                  \
      return RTLIB::LibcallPrefix##128;                                        \
    default:                                                                   \
      llvm_unreachable("unexpected size");                                     \
    }                                                                          \
  } while (0)

◆ RTLIBCASE_INT

#define RTLIBCASE_INT ( LibcallPrefix )

Value:

  do {                                                                         \
    switch (Size) {                                                            \
    case 32:                                                                   \
      return RTLIB::LibcallPrefix##32;                                         \
    case 64:                                                                   \
      return RTLIB::LibcallPrefix##64;                                         \
    case 128:                                                                  \
      return RTLIB::LibcallPrefix##128;                                        \
    default:                                                                   \
      llvm_unreachable("unexpected size");                                     \
    }                                                                          \
  } while (0)

Function Documentation

◆ broadcastSrcOp()

static void broadcastSrcOp	(	SmallVectorImpl< SrcOp > &	Ops,
		unsigned	N,
		MachineOperand &	Op
	)

static

Operand Op is used on N sub-instructions.

Fill Ops with N SrcOps made from Op depending on operand type.

Definition at line 3706 of file LegalizerHelper.cpp.

References llvm_unreachable, N, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType().

◆ buildBitFieldInsert()

static Register buildBitFieldInsert	(	MachineIRBuilder &	B,
		Register	TargetReg,
		Register	InsertReg,
		Register	OffsetBits
	)

static

Emit code to insert InsertReg into TargetRet at OffsetBits in TargetReg, while preserving other bits in TargetReg.

(InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)

Definition at line 2868 of file LegalizerHelper.cpp.

References B, llvm::APInt::getLowBitsSet(), and llvm::LLT::getSizeInBits().

Referenced by llvm::LegalizerHelper::bitcastInsertVectorElt().

◆ clampDynamicVectorIndex()

static Register clampDynamicVectorIndex	(	MachineIRBuilder &	B,
		Register	IdxReg,
		LLT	VecTy
	)

static

Definition at line 3606 of file LegalizerHelper.cpp.

References B, llvm::APInt::getLowBitsSet(), llvm::LLT::getNumElements(), llvm::LLT::getSizeInBits(), llvm::isPowerOf2_32(), llvm::Log2_32(), llvm::MIPatternMatch::m_ICst(), and llvm::MIPatternMatch::mi_match().

Referenced by llvm::LegalizerHelper::getVectorElementPointer(), and llvm::TargetLowering::getVectorSubVecPointer().

◆ conversionLibcall()

static LegalizerHelper::LegalizeResult conversionLibcall	(	MachineInstr &	MI,
		MachineIRBuilder &	MIRBuilder,
		Type *	ToType,
		Type *	FromType
	)

static

Definition at line 788 of file LegalizerHelper.cpp.

References llvm::createLibcall(), getConvRTLibDesc(), getReg(), llvm::LegalizeActions::Libcall, and MI.

Referenced by llvm::LegalizerHelper::libcall().

◆ findGISelOptimalMemOpLowering()

static bool findGISelOptimalMemOpLowering	(	std::vector< LLT > &	MemOps,
		unsigned	Limit,
		const MemOp &	Op,
		unsigned	DstAS,
		unsigned	SrcAS,
		const AttributeList &	FuncAttributes,
		const TargetLowering &	TLI
	)

static

Definition at line 7650 of file LegalizerHelper.cpp.

References llvm::TargetLoweringBase::allowsMisalignedMemoryAccesses(), assert(), llvm::bit_floor(), llvm::CallingConv::Fast, llvm::getMVTForLLT(), llvm::TargetLoweringBase::getOptimalMemOpLLT(), llvm::LLT::getSizeInBits(), llvm::LLT::getSizeInBytes(), llvm::LLT::isVector(), llvm::MachineMemOperand::MONone, llvm::LLT::scalar(), and Size.

◆ getBitcastWiderVectorElementOffset()

static Register getBitcastWiderVectorElementOffset	(	MachineIRBuilder &	B,
		Register	Idx,
		unsigned	NewEltSize,
		unsigned	OldEltSize
	)

static

Figure out the bit offset into a register when coercing a vector index for the wide element type.

This is only for the case when promoting vector to one with larger elements. offset_idx = G_AND idx, ~(-1 << Log2(DstEltSize / SrcEltSize)) offset_bits = G_SHL offset_idx, Log2(SrcEltSize)

Definition at line 2736 of file LegalizerHelper.cpp.

References B, llvm::APInt::getAllOnes(), llvm::LLT::getSizeInBits(), Idx, and llvm::Log2_32().

Referenced by llvm::LegalizerHelper::bitcastExtractVectorElt(), and llvm::LegalizerHelper::bitcastInsertVectorElt().

◆ getConvRTLibDesc()

static RTLIB::Libcall getConvRTLibDesc	(	unsigned	Opcode,
		Type *	ToType,
		Type *	FromType
	)

static

Definition at line 765 of file LegalizerHelper.cpp.

References llvm::RTLIB::getFPEXT(), llvm::RTLIB::getFPROUND(), llvm::RTLIB::getFPTOSINT(), llvm::RTLIB::getFPTOUINT(), llvm::RTLIB::getSINTTOFP(), llvm::RTLIB::getUINTTOFP(), llvm::MVT::getVT(), and llvm_unreachable.

Referenced by conversionLibcall().

◆ getFloatTypeForLLT()

static Type * getFloatTypeForLLT	(	LLVMContext &	Ctx,
		LLT	Ty
	)

static

Definition at line 81 of file LegalizerHelper.cpp.

References llvm::Type::getDoubleTy(), llvm::Type::getFloatTy(), llvm::Type::getFP128Ty(), llvm::Type::getHalfTy(), llvm::LLT::getSizeInBits(), llvm::Type::getX86_FP80Ty(), and llvm::LLT::isScalar().

Referenced by llvm::LegalizerHelper::libcall(), and llvm::LegalizerHelper::lowerFConstant().

◆ getMemsetValue()

static Register getMemsetValue	(	Register	Val,
		LLT	Ty,
		MachineIRBuilder &	MIB
	)

static

Definition at line 7723 of file LegalizerHelper.cpp.

References llvm::MachineIRBuilder::buildConstant(), llvm::MachineIRBuilder::buildMul(), llvm::MachineIRBuilder::buildSplatVector(), llvm::MachineIRBuilder::buildZExtOrTrunc(), llvm::getIConstantVRegValWithLookThrough(), llvm::MachineIRBuilder::getMRI(), llvm::MachineInstrBuilder::getReg(), llvm::LLT::getScalarSizeInBits(), llvm::LLT::getScalarType(), llvm::APInt::getSplat(), llvm::LLT::isVector(), and MRI.

Referenced by getMemsetStores().

◆ getNarrowTypeBreakDown()

static std::pair< int, int > getNarrowTypeBreakDown	(	LLT	OrigTy,
		LLT	NarrowTy,
		LLT &	LeftoverTy
	)

static

Try to break down OrigTy into NarrowTy sized pieces.

Returns the number of NarrowTy elements needed to reconstruct OrigTy, with any leftover piece as type LeftoverTy

Returns -1 in the first element of the pair if the breakdown is not satisfiable.

Definition at line 55 of file LegalizerHelper.cpp.

References assert(), llvm::ElementCount::getFixed(), llvm::LLT::getScalarSizeInBits(), llvm::LLT::getSizeInBits(), llvm::LLT::isValid(), llvm::LLT::isVector(), llvm::LLT::scalar(), llvm::LLT::scalarOrVector(), and Size.

Referenced by makeDstOps(), and llvm::LegalizerHelper::reduceLoadStoreWidth().

◆ getRTLibDesc()

static RTLIB::Libcall getRTLibDesc	(	unsigned	Opcode,
		unsigned	Size
	)

static

Definition at line 473 of file LegalizerHelper.cpp.

References llvm_unreachable, RTLIBCASE, and RTLIBCASE_INT.

Referenced by simpleLibcall().

◆ getScalarOpcForReduction()

static unsigned getScalarOpcForReduction ( unsigned Opc )

static

Definition at line 4447 of file LegalizerHelper.cpp.

References llvm_unreachable.

Referenced by llvm::LegalizerHelper::fewerElementsVectorReductions().

◆ getTypeForLLT()

static Type * getTypeForLLT	(	LLT	Ty,
		LLVMContext &	C
	)

static

Definition at line 7715 of file LegalizerHelper.cpp.

References llvm::CallingConv::C, llvm::IntegerType::get(), llvm::FixedVectorType::get(), llvm::LLT::getNumElements(), llvm::LLT::getScalarSizeInBits(), llvm::LLT::getSizeInBits(), and llvm::LLT::isVector().

◆ getUnmergePieces()

static void getUnmergePieces	(	SmallVectorImpl< Register > &	Pieces,
		MachineIRBuilder &	B,
		Register	Src,
		LLT	Ty
	)

static

Definition at line 2628 of file LegalizerHelper.cpp.

References B, E, I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::LegalizerHelper::lowerBitcast().

◆ getUnmergeResults()

static void getUnmergeResults	(	SmallVectorImpl< Register > &	Regs,
		const MachineInstr &	MI
	)

static

Append the result registers of G_UNMERGE_VALUES MI to Regs.

Definition at line 316 of file LegalizerHelper.cpp.

References assert(), I, MI, llvm::SmallVectorImpl< T >::resize(), and llvm::SmallVectorBase< Size_T >::size().

◆ hasSameNumEltsOnAllVectorOperands()

static bool hasSameNumEltsOnAllVectorOperands	(	GenericMachineInstr &	MI,
		MachineRegisterInfo &	MRI,
		std::initializer_list< unsigned >	NonVecOpIndices
	)

static

Check that all vector operands have same number of elements.

Other operands should be listed in NonVecOp.

Definition at line 3645 of file LegalizerHelper.cpp.

References llvm::LLT::getNumElements(), llvm::is_contained(), llvm::LLT::isVector(), MI, and MRI.

Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType().

◆ isLibCallInTailPosition()

static bool isLibCallInTailPosition	(	MachineInstr &	MI,
		const TargetInstrInfo &	TII,
		MachineRegisterInfo &	MRI
	)

static

True if an instruction is in tail position in its caller.

Intended for legalizing libcalls as tail calls when possible.

Definition at line 567 of file LegalizerHelper.cpp.

References F, llvm::MachineFunction::getFunction(), llvm::MachineBasicBlock::getParent(), llvm::AttributeList::getRetAttrs(), llvm::AttrBuilder::hasAttributes(), llvm::AttributeList::hasRetAttr(), llvm::MachineBasicBlock::instr_end(), llvm::Register::isPhysical(), llvm::HexagonInstrInfo::isTailCall(), llvm::Register::isVirtual(), llvm_unreachable, MBB, MI, llvm::next_nodbg(), llvm::AttrBuilder::removeAttribute(), and TII.

Referenced by llvm::createMemLibcall().

◆ isNonZeroModBitWidthOrUndef()

static bool isNonZeroModBitWidthOrUndef	(	const MachineRegisterInfo &	MRI,
		Register	Reg,
		unsigned	BW
	)

static

Definition at line 5840 of file LegalizerHelper.cpp.

References llvm::CallingConv::C, llvm::ConstantInt::getValue(), llvm::matchUnaryPredicate(), MRI, and llvm::APInt::urem().

Referenced by llvm::TargetLowering::expandFunnelShift(), expandVPFunnelShift(), llvm::LegalizerHelper::lowerFunnelShiftAsShifts(), and llvm::LegalizerHelper::lowerFunnelShiftWithInverse().

◆ makeDstOps()

static void makeDstOps	(	SmallVectorImpl< DstOp > &	DstOps,
		LLT	Ty,
		unsigned	NumElts
	)

static

Fill DstOps with DstOps that have same number of elements combined as the Ty.

These DstOps have either scalar type when NumElts = 1 or are vectors with NumElts elements. When Ty.getNumElements() is not multiple of NumElts last DstOp (leftover) has fewer then NumElts elements.

Definition at line 3683 of file LegalizerHelper.cpp.

References assert(), llvm::LLT::fixed_vector(), llvm::LLT::getElementType(), getNarrowTypeBreakDown(), llvm::LLT::isValid(), llvm::LLT::isVector(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType(), and llvm::LegalizerHelper::fewerElementsVectorPhi().