#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/Analysis.h"
#include "llvm/CodeGen/CallingConvLower.h"
#include "llvm/CodeGen/CodeGenCommonISel.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SDPatternMatch.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/Support/DivisionByConstantInfo.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetMachine.h"
#include <cctype>
#include <deque>

Functions
static SDValue	combineShiftToAVG (SDValue Op, TargetLowering::TargetLoweringOpt &TLO, const TargetLowering &TLI, const APInt &DemandedBits, const APInt &DemandedElts, unsigned Depth)
static APInt	getKnownUndefForVectorBinop (SDValue BO, SelectionDAG &DAG, const APInt &UndefOp0, const APInt &UndefOp1)
	Given a vector binary operation and known undefined elements for each input operand, compute whether each element of the output is undefined.
static SDValue	simplifySetCCWithCTPOP (const TargetLowering &TLI, EVT VT, SDValue N0, const APInt &C1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static SDValue	foldSetCCWithRotate (EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static SDValue	foldSetCCWithFunnelShift (EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, const SDLoc &dl, SelectionDAG &DAG)
static unsigned	getConstraintPiority (TargetLowering::ConstraintType CT)
	Return a number indicating our preference for chosing a type of constraint over another, for the purpose of sorting them.
static bool	lowerImmediateIfPossible (TargetLowering::ConstraintPair &P, SDValue Op, SelectionDAG *DAG, const TargetLowering &TLI)
	If we have an immediate, see if we can lower it.
static SDValue	BuildExactSDIV (const TargetLowering &TLI, SDNode N, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDNode > &Created)
	Given an exact SDIV by a constant, create a multiplication with the multiplicative inverse of the constant.
static SDValue	BuildExactUDIV (const TargetLowering &TLI, SDNode N, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDNode > &Created)
	Given an exact UDIV by a constant, create a multiplication with the multiplicative inverse of the constant.
static void	turnVectorIntoSplatVector (MutableArrayRef< SDValue > Values, std::function< bool(SDValue)> Predicate, SDValue AlternativeReplacement=SDValue())
	If all values in Values that don't match the predicate are same 'splat' value, then replace all values with that splat value.
static bool	isNonZeroModBitWidthOrUndef (SDValue Z, unsigned BW)
static SDValue	expandVPFunnelShift (SDNode *Node, SelectionDAG &DAG)
static std::optional< bool >	isFCmpEqualZero (FPClassTest Test, const fltSemantics &Semantics, const MachineFunction &MF)
	Returns a true value if if this FPClassTest can be performed with an ordered fcmp to 0, and a false value if it's an unordered fcmp to 0.
static bool	canExpandVectorCTPOP (const TargetLowering &TLI, EVT VT)
static SDValue	clampDynamicVectorIndex (SelectionDAG &DAG, SDValue Idx, EVT VecVT, const SDLoc &dl, ElementCount SubEC)
static bool	canFoldStoreIntoLibCallOutputPointers (StoreSDNode StoreNode, SDNode FPNode)
	Given a store node `StoreNode`, return true if it is safe to fold that node into `FPNode`, which expands to a library call with output pointers.

Function Documentation

◆ BuildExactSDIV()

SDValue BuildExactSDIV	(	const TargetLowering &	TLI,
		SDNode *	N,
		const SDLoc &	dl,
		SelectionDAG &	DAG,
		SmallVectorImpl< SDNode * > &	Created )

static

Given an exact SDIV by a constant, create a multiplication with the multiplicative inverse of the constant.

Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242

Definition at line 6340 of file TargetLowering.cpp.

References llvm::APInt::ashrInPlace(), assert(), llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, llvm::APInt::countr_zero(), llvm::SDNodeFlags::Exact, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getScalarType(), llvm::TargetLoweringBase::getShiftAmountTy(), llvm::SelectionDAG::getSplatVector(), llvm::isa(), llvm::ISD::matchUnaryPredicate(), llvm::ISD::MUL, llvm::APInt::multiplicativeInverse(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::SPLAT_VECTOR, and llvm::ISD::SRA.

Referenced by llvm::TargetLowering::BuildSDIV().

◆ BuildExactUDIV()

SDValue BuildExactUDIV	(	const TargetLowering &	TLI,
		SDNode *	N,
		const SDLoc &	dl,
		SelectionDAG &	DAG,
		SmallVectorImpl< SDNode * > &	Created )

static

Given an exact UDIV by a constant, create a multiplication with the multiplicative inverse of the constant.

Ref: "Hacker's Delight" by Henry Warren, 2nd Edition, p. 242

Definition at line 6402 of file TargetLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, llvm::CallingConv::C, llvm::APInt::countr_zero(), llvm::SDNodeFlags::Exact, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getDataLayout(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::EVT::getScalarType(), llvm::TargetLoweringBase::getShiftAmountTy(), llvm::SelectionDAG::getSplatVector(), llvm::isa(), llvm::APInt::lshrInPlace(), llvm::ISD::matchUnaryPredicate(), llvm::ISD::MUL, llvm::APInt::multiplicativeInverse(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SDValue(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::SPLAT_VECTOR, and llvm::ISD::SRL.

Referenced by llvm::TargetLowering::BuildUDIV().

◆ canExpandVectorCTPOP()

bool canExpandVectorCTPOP	(	const TargetLowering &	TLI,
		EVT	VT )

static

Definition at line 9265 of file TargetLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, assert(), llvm::EVT::getScalarSizeInBits(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::TargetLoweringBase::isOperationLegalOrCustomOrPromote(), llvm::EVT::isVector(), llvm::ISD::MUL, llvm::ISD::SRL, and llvm::ISD::SUB.

Referenced by llvm::TargetLowering::expandCTLZ(), llvm::TargetLowering::expandCTPOP(), and llvm::TargetLowering::expandCTTZ().

◆ canFoldStoreIntoLibCallOutputPointers()

bool canFoldStoreIntoLibCallOutputPointers	(	StoreSDNode *	StoreNode,
		SDNode *	FPNode )

static

Given a store node StoreNode, return true if it is safe to fold that node into FPNode, which expands to a library call with output pointers.

Definition at line 12139 of file TargetLowering.cpp.

References _, llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SelectionDAG::getHasPredecessorMaxSteps(), llvm::SDNode::hasPredecessorHelper(), llvm::SmallPtrSetImpl< PtrType >::insert(), MaxSteps, llvm::SDNode::ops(), llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SmallPtrSetImplBase::size().

Referenced by llvm::TargetLowering::expandMultipleResultFPLibCall().

◆ clampDynamicVectorIndex()

SDValue clampDynamicVectorIndex	(	SelectionDAG &	DAG,
		SDValue	Idx,
		EVT	VecVT,
		const SDLoc &	dl,
		ElementCount	SubEC )

static

◆ combineShiftToAVG()

SDValue combineShiftToAVG	(	SDValue	Op,
		TargetLowering::TargetLoweringOpt &	TLO,
		const TargetLowering &	TLI,
		const APInt &	DemandedBits,
		const APInt &	DemandedElts,
		unsigned	Depth )

static

Definition at line 1013 of file TargetLowering.cpp.

References A(), llvm::ISD::ADD, llvm::Add, assert(), llvm::ISD::AVGCEILS, llvm::ISD::AVGCEILU, llvm::ISD::AVGFLOORS, llvm::ISD::AVGFLOORU, llvm::bit_ceil(), llvm::SelectionDAG::computeKnownBits(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::KnownBits::countMinLeadingZeros(), llvm::TargetLowering::TargetLoweringOpt::DAG, llvm::Depth, DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtOrTrunc(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::isa(), llvm::isConstOrConstSplat(), llvm::ConstantSDNode::isOne(), llvm::TargetLoweringBase::isOperationLegal(), llvm::EVT::isVector(), llvm::TargetLowering::TargetLoweringOpt::LegalOperations(), llvm::TargetLowering::TargetLoweringOpt::LegalTypes(), llvm_unreachable, SDValue(), llvm::ISD::SRA, llvm::ISD::SRL, and llvm::SelectionDAG::willNotOverflowAdd().

Referenced by llvm::TargetLowering::SimplifyDemandedBits().

◆ expandVPFunnelShift()

SDValue expandVPFunnelShift	(	SDNode *	Node,
		SelectionDAG &	DAG )

static

Definition at line 8167 of file TargetLowering.cpp.

References DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), isNonZeroModBitWidthOrUndef(), llvm::isPowerOf2_32(), SDValue(), X, and Y.

Referenced by llvm::TargetLowering::expandFunnelShift().

◆ foldSetCCWithFunnelShift()

SDValue foldSetCCWithFunnelShift	(	EVT	VT,
		SDValue	N0,
		SDValue	N1,
		ISD::CondCode	Cond,
		const SDLoc &	dl,
		SelectionDAG &	DAG )

static

◆ foldSetCCWithRotate()

SDValue foldSetCCWithRotate	(	EVT	VT,
		SDValue	N0,
		SDValue	N1,
		ISD::CondCode	Cond,
		const SDLoc &	dl,
		SelectionDAG &	DAG )

static

◆ getConstraintPiority()

unsigned getConstraintPiority ( TargetLowering::ConstraintType CT )

static

Return a number indicating our preference for chosing a type of constraint over another, for the purpose of sorting them.

Immediates are almost always preferrable (when they can be emitted). A higher return value means a stronger preference for one constraint type relative to another. FIXME: We should prefer registers over memory but doing so may lead to unrecoverable register exhaustion later. https://github.com/llvm/llvm-project/issues/20571

Definition at line 6117 of file TargetLowering.cpp.

References llvm::TargetLowering::C_Address, llvm::TargetLowering::C_Immediate, llvm::TargetLowering::C_Memory, llvm::TargetLowering::C_Other, llvm::TargetLowering::C_Register, llvm::TargetLowering::C_RegisterClass, llvm::TargetLowering::C_Unknown, and llvm_unreachable.

Referenced by llvm::TargetLowering::getConstraintPreferences().

◆ getKnownUndefForVectorBinop()

APInt getKnownUndefForVectorBinop	(	SDValue	BO,
		SelectionDAG &	DAG,
		const APInt &	UndefOp0,
		const APInt &	UndefOp1 )

static

Given a vector binary operation and known undefined elements for each input operand, compute whether each element of the output is undefined.

Definition at line 3123 of file TargetLowering.cpp.

References assert(), llvm::CallingConv::C, llvm::dyn_cast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), llvm::isa(), llvm::TargetLoweringBase::isBinOp(), llvm::EVT::isFixedLengthVector(), isUndef(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), SDValue(), and llvm::APInt::setBit().

Referenced by llvm::TargetLowering::SimplifyDemandedVectorElts().

◆ isFCmpEqualZero()

std::optional< bool > isFCmpEqualZero	(	FPClassTest	Test,
		const fltSemantics &	Semantics,
		const MachineFunction &	MF )

static

Returns a true value if if this FPClassTest can be performed with an ordered fcmp to 0, and a false value if it's an unordered fcmp to 0.

Returns std::nullopt if it cannot be performed as a compare with 0.

Definition at line 8895 of file TargetLowering.cpp.

References llvm::fcNan, llvm::fcNone, llvm::fcSubnormal, llvm::fcZero, llvm::MachineFunction::getDenormalMode(), llvm::DenormalMode::IEEE, llvm::DenormalMode::Input, llvm::DenormalMode::inputsAreZero(), and llvm::Test.

Referenced by llvm::TargetLowering::expandIS_FPCLASS().

◆ isNonZeroModBitWidthOrUndef()

bool isNonZeroModBitWidthOrUndef	(	SDValue	Z,
		unsigned	BW )

static

Definition at line 8160 of file TargetLowering.cpp.

References llvm::CallingConv::C, and llvm::ISD::matchUnaryPredicate().

◆ lowerImmediateIfPossible()

bool lowerImmediateIfPossible	(	TargetLowering::ConstraintPair &	P,
		SDValue	Op,
		SelectionDAG *	DAG,
		const TargetLowering &	TLI )

static

If we have an immediate, see if we can lower it.

Return true if we can, false otherwise.

Definition at line 6263 of file TargetLowering.cpp.

References assert(), llvm::TargetLowering::C_Immediate, llvm::TargetLowering::C_Other, llvm::TargetLowering::LowerAsmOperandForConstraint(), and P.

Referenced by llvm::TargetLowering::ComputeConstraintToUse().

◆ simplifySetCCWithCTPOP()

SDValue simplifySetCCWithCTPOP	(	const TargetLowering &	TLI,
		EVT	VT,
		SDValue	N0,
		const APInt &	C1,
		ISD::CondCode	Cond,
		const SDLoc &	dl,
		SelectionDAG &	DAG )

static

Definition at line 4466 of file TargetLowering.cpp.

References llvm::ISD::ADD, llvm::Add, llvm::ISD::AND, assert(), Cond, llvm::ISD::CTPOP, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getConstant(), llvm::TargetLoweringBase::getCustomCtpopCost(), llvm::APInt::getLimitedValue(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::hasOneUse(), llvm::TargetLoweringBase::isCtpopFast(), llvm::EVT::isInteger(), llvm::SelectionDAG::isKnownNeverZero(), llvm::EVT::isVector(), llvm::Log2_32(), Passes, RHS, SDValue(), llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::TRUNCATE, llvm::APInt::ugt(), llvm::ISD::XOR, and llvm::Xor.

Referenced by llvm::TargetLowering::SimplifySetCC().

◆ turnVectorIntoSplatVector()

void turnVectorIntoSplatVector	(	MutableArrayRef< SDValue >	Values,
		std::function< bool(SDValue)>	Predicate,
		SDValue	AlternativeReplacement = SDValue() )

static

If all values in Values that don't match the predicate are same 'splat' value, then replace all values with that splat value.

Else, if AlternativeReplacement was provided, then replace all values that do match predicate with AlternativeReplacement value.

Definition at line 6902 of file TargetLowering.cpp.

References llvm::all_of(), llvm::MutableArrayRef< T >::begin(), llvm::MutableArrayRef< T >::end(), and llvm::find_if_not().

Functions

Function Documentation

◆ BuildExactSDIV()

◆ BuildExactUDIV()

◆ canExpandVectorCTPOP()

◆ canFoldStoreIntoLibCallOutputPointers()

◆ clampDynamicVectorIndex()

◆ combineShiftToAVG()

◆ expandVPFunnelShift()

◆ foldSetCCWithFunnelShift()

◆ foldSetCCWithRotate()

◆ getConstraintPiority()

◆ getKnownUndefForVectorBinop()

◆ isFCmpEqualZero()

◆ isNonZeroModBitWidthOrUndef()

◆ lowerImmediateIfPossible()

◆ simplifySetCCWithCTPOP()

◆ turnVectorIntoSplatVector()