#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"
#include "llvm/CodeGen/GlobalISel/CallLowering.h"
#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"
#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"
#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
#include "llvm/CodeGen/GlobalISel/Utils.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetFrameLowering.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/Instructions.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include <numeric>
#include <optional>

Include dependency graph for LegalizerHelper.cpp:

Macros
#define	DEBUG_TYPE "legalizer"

#define	RTLIBCASE_INT(LibcallPrefix)

#define	RTLIBCASE(LibcallPrefix)

Functions
static std::pair< int, int >	getNarrowTypeBreakDown (LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy)
	Try to break down `OrigTy` into `NarrowTy` sized pieces. More...

static Type *	getFloatTypeForLLT (LLVMContext &Ctx, LLT Ty)

static void	getUnmergeResults (SmallVectorImpl< Register > &Regs, const MachineInstr &MI)
	Append the result registers of G_UNMERGE_VALUES `MI` to `Regs`. More...

static RTLIB::Libcall	getRTLibDesc (unsigned Opcode, unsigned Size)

static bool	isLibCallInTailPosition (MachineInstr &MI, const TargetInstrInfo &TII, MachineRegisterInfo &MRI)
	True if an instruction is in tail position in its caller. More...

static LegalizerHelper::LegalizeResult	simpleLibcall (MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType)

static RTLIB::Libcall	getConvRTLibDesc (unsigned Opcode, Type ToType, Type FromType)

static LegalizerHelper::LegalizeResult	conversionLibcall (MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type ToType, Type FromType)

static void	getUnmergePieces (SmallVectorImpl< Register > &Pieces, MachineIRBuilder &B, Register Src, LLT Ty)

static Register	getBitcastWiderVectorElementOffset (MachineIRBuilder &B, Register Idx, unsigned NewEltSize, unsigned OldEltSize)
	Figure out the bit offset into a register when coercing a vector index for the wide element type. More...

static Register	buildBitFieldInsert (MachineIRBuilder &B, Register TargetReg, Register InsertReg, Register OffsetBits)
	Emit code to insert `InsertReg` into `TargetRet` at `OffsetBits` in `TargetReg`, while preserving other bits in `TargetReg`. More...

static Register	clampDynamicVectorIndex (MachineIRBuilder &B, Register IdxReg, LLT VecTy)

static bool	hasSameNumEltsOnAllVectorOperands (GenericMachineInstr &MI, MachineRegisterInfo &MRI, std::initializer_list< unsigned > NonVecOpIndices)
	Check that all vector operands have same number of elements. More...

static void	makeDstOps (SmallVectorImpl< DstOp > &DstOps, LLT Ty, unsigned NumElts)
	Fill `DstOps` with DstOps that have same number of elements combined as the Ty. More...

static void	broadcastSrcOp (SmallVectorImpl< SrcOp > &Ops, unsigned N, MachineOperand &Op)
	Operand `Op` is used on `N` sub-instructions. More...

static unsigned	getScalarOpcForReduction (unsigned Opc)

static bool	isNonZeroModBitWidthOrUndef (const MachineRegisterInfo &MRI, Register Reg, unsigned BW)

static CmpInst::Predicate	minMaxToCompare (unsigned Opc)

static MachineInstrBuilder	SwapN (unsigned N, DstOp Dst, MachineIRBuilder &B, MachineInstrBuilder Src, APInt Mask)

static bool	shouldLowerMemFuncForSize (const MachineFunction &MF)

static bool	findGISelOptimalMemOpLowering (std::vector< LLT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, const TargetLowering &TLI)

static Type *	getTypeForLLT (LLT Ty, LLVMContext &C)

static Register	getMemsetValue (Register Val, LLT Ty, MachineIRBuilder &MIB)

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "legalizer"

Definition at line 39 of file LegalizerHelper.cpp.

◆ RTLIBCASE

#define RTLIBCASE ( LibcallPrefix )

Value:

  do {                                                                         \
    switch (Size) {                                                            \
    case 32:                                                                   \
      return RTLIB::LibcallPrefix##32;                                         \
    case 64:                                                                   \
      return RTLIB::LibcallPrefix##64;                                         \
    case 80:                                                                   \
      return RTLIB::LibcallPrefix##80;                                         \
    case 128:                                                                  \
      return RTLIB::LibcallPrefix##128;                                        \
    default:                                                                   \
      llvm_unreachable("unexpected size");                                     \
    }                                                                          \
  } while (0)

◆ RTLIBCASE_INT

#define RTLIBCASE_INT ( LibcallPrefix )

Value:

  do {                                                                         \
    switch (Size) {                                                            \
    case 32:                                                                   \
      return RTLIB::LibcallPrefix##32;                                         \
    case 64:                                                                   \
      return RTLIB::LibcallPrefix##64;                                         \
    case 128:                                                                  \
      return RTLIB::LibcallPrefix##128;                                        \
    default:                                                                   \
      llvm_unreachable("unexpected size");                                     \
    }                                                                          \
  } while (0)

Function Documentation

◆ broadcastSrcOp()

static void broadcastSrcOp	(	SmallVectorImpl< SrcOp > &	Ops,
		unsigned	N,
		MachineOperand &	Op
	)

static

Operand Op is used on N sub-instructions.

Fill Ops with N SrcOps made from Op depending on operand type.

Definition at line 3674 of file LegalizerHelper.cpp.

References i, llvm_unreachable, and N.

Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType().

◆ buildBitFieldInsert()

static Register buildBitFieldInsert	(	MachineIRBuilder &	B,
		Register	TargetReg,
		Register	InsertReg,
		Register	OffsetBits
	)

static

Emit code to insert InsertReg into TargetRet at OffsetBits in TargetReg, while preserving other bits in TargetReg.

(InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)

Definition at line 2838 of file LegalizerHelper.cpp.

References B, llvm::APInt::getLowBitsSet(), and llvm::LLT::getSizeInBits().

Referenced by llvm::LegalizerHelper::bitcastInsertVectorElt().

◆ clampDynamicVectorIndex()

static Register clampDynamicVectorIndex	(	MachineIRBuilder &	B,
		Register	IdxReg,
		LLT	VecTy
	)

static

Definition at line 3574 of file LegalizerHelper.cpp.

References B, llvm::APInt::getLowBitsSet(), llvm::LLT::getNumElements(), llvm::LLT::getSizeInBits(), llvm::RISCVMatInt::Imm, llvm::isPowerOf2_32(), llvm::Log2_32(), llvm::MIPatternMatch::m_ICst(), and llvm::MIPatternMatch::mi_match().

Referenced by llvm::LegalizerHelper::getVectorElementPointer().

◆ conversionLibcall()

static LegalizerHelper::LegalizeResult conversionLibcall	(	MachineInstr &	MI,
		MachineIRBuilder &	MIRBuilder,
		Type *	ToType,
		Type *	FromType
	)

static

Definition at line 785 of file LegalizerHelper.cpp.

References llvm::createLibcall(), getConvRTLibDesc(), getReg(), and MI.

Referenced by llvm::LegalizerHelper::libcall().

◆ findGISelOptimalMemOpLowering()

static bool findGISelOptimalMemOpLowering	(	std::vector< LLT > &	MemOps,
		unsigned	Limit,
		const MemOp &	Op,
		unsigned	DstAS,
		unsigned	SrcAS,
		const AttributeList &	FuncAttributes,
		const TargetLowering &	TLI
	)

static

Definition at line 7535 of file LegalizerHelper.cpp.

References llvm::TargetLoweringBase::allowsMisalignedMemoryAccesses(), assert(), llvm::getMVTForLLT(), llvm::TargetLoweringBase::getOptimalMemOpLLT(), llvm::LLT::getSizeInBits(), llvm::LLT::getSizeInBytes(), llvm::LLT::isVector(), llvm::MachineMemOperand::MONone, llvm::PowerOf2Floor(), and llvm::LLT::scalar().

◆ getBitcastWiderVectorElementOffset()

static Register getBitcastWiderVectorElementOffset	(	MachineIRBuilder &	B,
		Register	Idx,
		unsigned	NewEltSize,
		unsigned	OldEltSize
	)

static

Figure out the bit offset into a register when coercing a vector index for the wide element type.

This is only for the case when promoting vector to one with larger elements. offset_idx = G_AND idx, ~(-1 << Log2(DstEltSize / SrcEltSize)) offset_bits = G_SHL offset_idx, Log2(SrcEltSize)

Definition at line 2706 of file LegalizerHelper.cpp.

References B, llvm::APInt::getAllOnes(), llvm::LLT::getSizeInBits(), and llvm::Log2_32().

Referenced by llvm::LegalizerHelper::bitcastExtractVectorElt(), and llvm::LegalizerHelper::bitcastInsertVectorElt().

◆ getConvRTLibDesc()

static RTLIB::Libcall getConvRTLibDesc	(	unsigned	Opcode,
		Type *	ToType,
		Type *	FromType
	)

static

Definition at line 762 of file LegalizerHelper.cpp.

References llvm::RTLIB::getFPEXT(), llvm::RTLIB::getFPROUND(), llvm::RTLIB::getFPTOSINT(), llvm::RTLIB::getFPTOUINT(), llvm::RTLIB::getSINTTOFP(), llvm::RTLIB::getUINTTOFP(), llvm::MVT::getVT(), and llvm_unreachable.

Referenced by conversionLibcall().

◆ getFloatTypeForLLT()

static Type* getFloatTypeForLLT	(	LLVMContext &	Ctx,
		LLT	Ty
	)

static

Definition at line 79 of file LegalizerHelper.cpp.

References llvm::Type::getDoubleTy(), llvm::Type::getFloatTy(), llvm::Type::getFP128Ty(), llvm::Type::getHalfTy(), llvm::LLT::getSizeInBits(), llvm::Type::getX86_FP80Ty(), and llvm::LLT::isScalar().

Referenced by llvm::LegalizerHelper::libcall().

◆ getMemsetValue()

static Register getMemsetValue	(	Register	Val,
		LLT	Ty,
		MachineIRBuilder &	MIB
	)

static

Definition at line 7608 of file LegalizerHelper.cpp.

References llvm::MachineIRBuilder::buildConstant(), llvm::MachineIRBuilder::buildMul(), llvm::MachineIRBuilder::buildSplatVector(), llvm::MachineIRBuilder::buildZExtOrTrunc(), llvm::getIConstantVRegValWithLookThrough(), llvm::MachineIRBuilder::getMRI(), llvm::MachineInstrBuilder::getReg(), llvm::LLT::getScalarSizeInBits(), llvm::LLT::getScalarType(), llvm::APInt::getSplat(), llvm::LLT::isVector(), llvm::msf::Magic, and MRI.

◆ getNarrowTypeBreakDown()

static std::pair<int, int> getNarrowTypeBreakDown	(	LLT	OrigTy,
		LLT	NarrowTy,
		LLT &	LeftoverTy
	)

static

Try to break down OrigTy into NarrowTy sized pieces.

Returns the number of NarrowTy elements needed to reconstruct OrigTy, with any leftover piece as type LeftoverTy

Returns -1 in the first element of the pair if the breakdown is not satisfiable.

Definition at line 53 of file LegalizerHelper.cpp.

References assert(), llvm::LinearPolySize< ElementCount >::getFixed(), llvm::LLT::getScalarSizeInBits(), llvm::LLT::getSizeInBits(), llvm::LLT::isValid(), llvm::LLT::isVector(), llvm::LLT::scalar(), and llvm::LLT::scalarOrVector().

Referenced by makeDstOps(), and llvm::LegalizerHelper::reduceLoadStoreWidth().

◆ getRTLibDesc()

static RTLIB::Libcall getRTLibDesc	(	unsigned	Opcode,
		unsigned	Size
	)

static

Definition at line 470 of file LegalizerHelper.cpp.

References llvm_unreachable, RTLIBCASE, and RTLIBCASE_INT.

Referenced by simpleLibcall().

◆ getScalarOpcForReduction()

static unsigned getScalarOpcForReduction ( unsigned Opc )

static

Definition at line 4413 of file LegalizerHelper.cpp.

References llvm_unreachable.

Referenced by llvm::LegalizerHelper::fewerElementsVectorReductions().

◆ getTypeForLLT()

static Type* getTypeForLLT	(	LLT	Ty,
		LLVMContext &	C
	)

static

Definition at line 7600 of file LegalizerHelper.cpp.

References llvm::IntegerType::get(), llvm::FixedVectorType::get(), llvm::LLT::getNumElements(), llvm::LLT::getScalarSizeInBits(), llvm::LLT::getSizeInBits(), and llvm::LLT::isVector().

◆ getUnmergePieces()

static void getUnmergePieces	(	SmallVectorImpl< Register > &	Pieces,
		MachineIRBuilder &	B,
		Register	Src,
		LLT	Ty
	)

static

Definition at line 2624 of file LegalizerHelper.cpp.

References B, E, and I.

Referenced by llvm::LegalizerHelper::lowerBitcast().

◆ getUnmergeResults()

static void getUnmergeResults	(	SmallVectorImpl< Register > &	Regs,
		const MachineInstr &	MI
	)

static

Append the result registers of G_UNMERGE_VALUES MI to Regs.

Definition at line 313 of file LegalizerHelper.cpp.

References assert(), I, MI, and llvm::SmallVectorImpl< T >::resize().

◆ hasSameNumEltsOnAllVectorOperands()

static bool hasSameNumEltsOnAllVectorOperands	(	GenericMachineInstr &	MI,
		MachineRegisterInfo &	MRI,
		std::initializer_list< unsigned >	NonVecOpIndices
	)

static

Check that all vector operands have same number of elements.

Other operands should be listed in NonVecOp.

Definition at line 3613 of file LegalizerHelper.cpp.

References llvm::LLT::getNumElements(), llvm::MachineRegisterInfo::getType(), llvm::is_contained(), llvm::LLT::isVector(), MI, and MRI.

Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType().

◆ isLibCallInTailPosition()

static bool isLibCallInTailPosition	(	MachineInstr &	MI,
		const TargetInstrInfo &	TII,
		MachineRegisterInfo &	MRI
	)

static

True if an instruction is in tail position in its caller.

Intended for legalizing libcalls as tail calls when possible.

Definition at line 564 of file LegalizerHelper.cpp.

References F, llvm::MachineFunction::getFunction(), llvm::MachineBasicBlock::getParent(), llvm::AttributeList::getRetAttrs(), llvm::AttrBuilder::hasAttributes(), llvm::AttributeList::hasRetAttr(), llvm::MachineBasicBlock::instr_end(), llvm::Register::isPhysical(), llvm::HexagonInstrInfo::isTailCall(), llvm::Register::isVirtual(), llvm_unreachable, MBB, MI, llvm::next_nodbg(), llvm::AttrBuilder::removeAttribute(), llvm::MipsISD::Ret, and TII.

Referenced by llvm::createMemLibcall().

◆ isNonZeroModBitWidthOrUndef()

static bool isNonZeroModBitWidthOrUndef	(	const MachineRegisterInfo &	MRI,
		Register	Reg,
		unsigned	BW
	)

static

Definition at line 5725 of file LegalizerHelper.cpp.

References llvm::ConstantInt::getValue(), llvm::matchUnaryPredicate(), MRI, and llvm::APInt::urem().

Referenced by llvm::LegalizerHelper::lowerFunnelShiftAsShifts(), and llvm::LegalizerHelper::lowerFunnelShiftWithInverse().

◆ makeDstOps()

static void makeDstOps	(	SmallVectorImpl< DstOp > &	DstOps,
		LLT	Ty,
		unsigned	NumElts
	)

static

Fill DstOps with DstOps that have same number of elements combined as the Ty.

These DstOps have either scalar type when NumElts = 1 or are vectors with NumElts elements. When Ty.getNumElements() is not multiple of NumElts last DstOp (leftover) has fewer then NumElts elements.

Definition at line 3651 of file LegalizerHelper.cpp.

References assert(), llvm::LLT::fixed_vector(), llvm::LLT::getElementType(), getNarrowTypeBreakDown(), i, llvm::LLT::isValid(), and llvm::LLT::isVector().

Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType(), and llvm::LegalizerHelper::fewerElementsVectorPhi().