This file implements transforms for initial VPlan construction. More...

#include "LoopVectorizationPlanner.h"
#include "VPlan.h"
#include "VPlanAnalysis.h"
#include "VPlanCFG.h"
#include "VPlanDominatorTree.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanTransforms.h"
#include "VPlanUtils.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/LoopVersioning.h"

Macros
#define	DEBUG_TYPE "vplan"

Functions
static bool	isHeaderBB (BasicBlock BB, Loop L)
static bool	canonicalHeaderAndLatch (VPBlockBase *HeaderVPB, const VPDominatorTree &VPDT)
	Checks if `HeaderVPB` is a loop header block in the plain CFG; that is, it has exactly 2 predecessors (preheader and latch), where the block dominates the latch and the preheader dominates the block.
static void	createLoopRegion (VPlan &Plan, VPBlockBase *HeaderVPB)
	Create a new VPRegionBlock for the loop starting at `HeaderVPB`.
static void	addCanonicalIVRecipes (VPlan &Plan, VPBasicBlock HeaderVPBB, VPBasicBlock LatchVPBB, Type *IdxTy, DebugLoc DL)
static void	createExtractsForLiveOuts (VPlan &Plan, VPBasicBlock *MiddleVPBB)
	Creates extracts for values in `Plan` defined in a loop region and used outside a loop region.
static void	addInitialSkeleton (VPlan &Plan, Type InductionTy, DebugLoc IVDL, PredicatedScalarEvolution &PSE, Loop TheLoop)
static void	simplifyLiveInsWithSCEV (VPlan &Plan, PredicatedScalarEvolution &PSE)
	Check `Plan's` live-in and replace them with constants, if they can be simplified via SCEV.
static void	printAfterInitialConstruction (VPlan &)
	To make RUN_VPLAN_PASS print initial VPlan.
static VPHeaderPHIRecipe *	createWidenInductionRecipe (PHINode Phi, VPPhi PhiR, VPIRValue *Start, const InductionDescriptor &IndDesc, VPlan &Plan, PredicatedScalarEvolution &PSE, Loop &OrigLoop, DebugLoc DL)
	Creates a VPWidenIntOrFpInductionRecipe or VPWidenPointerInductionRecipe for `Phi` based on `IndDesc`.
static void	insertCheckBlockBeforeVectorLoop (VPlan &Plan, VPBasicBlock *CheckBlockVPBB)
	Insert `CheckBlockVPBB` on the edge leading to the vector preheader, connecting it to both vector and scalar preheaders.
static void	addBypassBranch (VPlan &Plan, VPBasicBlock CheckBlockVPBB, VPValue Cond, bool AddBranchWeights)
	Create a BranchOnCond terminator in `CheckBlockVPBB`.
static VPInstruction *	findFindIVSelect (VPValue *BackedgeVal)
	Find and return the final select instruction of the FindIV result pattern for the given `BackedgeVal:` select(icmp ne ComputeReductionResult(ReducedIV), Sentinel), ComputeReductionResult(ReducedIV), Start.

Variables
static constexpr uint32_t	CheckBypassWeights [] = {1, 127}

Detailed Description

This file implements transforms for initial VPlan construction.

Definition in file VPlanConstruction.cpp.

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "vplan"

Definition at line 33 of file VPlanConstruction.cpp.

Function Documentation

◆ addBypassBranch()

void addBypassBranch	(	VPlan &	Plan,
		VPBasicBlock *	CheckBlockVPBB,
		VPValue *	Cond,
		bool	AddBranchWeights )

static

Create a BranchOnCond terminator in CheckBlockVPBB.

Optionally adds branch weights.

Definition at line 995 of file VPlanConstruction.cpp.

References llvm::VPInstruction::BranchOnCond, CheckBypassWeights, Cond, llvm::MDBuilder::createBranchWeights(), llvm::VPBuilder::createNaryOp(), DL, llvm::VPRegionBlock::getCanonicalIV(), llvm::VPlan::getContext(), llvm::VPRecipeBase::getDebugLoc(), and llvm::VPlan::getVectorLoopRegion().

Referenced by llvm::VPlanTransforms::attachCheckBlock().

◆ addCanonicalIVRecipes()

void addCanonicalIVRecipes	(	VPlan &	Plan,
		VPBasicBlock *	HeaderVPBB,
		VPBasicBlock *	LatchVPBB,
		Type *	IdxTy,
		DebugLoc	DL )

static

Definition at line 457 of file VPlanConstruction.cpp.

References llvm::VPBasicBlock::back(), llvm::VPBasicBlock::begin(), llvm::VPInstruction::BranchOnCount, DL, llvm::VPBasicBlock::empty(), llvm::VPRecipeBase::eraseFromParent(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPlan::getOrAddLiveIn(), llvm::VPBasicBlock::getTerminator(), llvm::VPlan::getVectorTripCount(), llvm::VPlan::getVFxUF(), llvm::VPBasicBlock::insert(), llvm::VPlanPatternMatch::m_BranchOnCond(), and llvm::PatternMatch::match().

Referenced by addInitialSkeleton().

◆ addInitialSkeleton()

void addInitialSkeleton	(	VPlan &	Plan,
		Type *	InductionTy,
		DebugLoc	IVDL,
		PredicatedScalarEvolution &	PSE,
		Loop *	TheLoop )

static

Definition at line 514 of file VPlanConstruction.cpp.

References addCanonicalIVRecipes(), llvm::VPUser::addOperand(), assert(), canonicalHeaderAndLatch(), llvm::cast(), llvm::VPBlockUtils::connectBlocks(), createExtractsForLiveOuts(), llvm::VPBuilder::createScalarPhi(), llvm::VPlan::createVPBasicBlock(), llvm::drop_begin(), llvm::VPlan::getEntry(), llvm::vputils::getOrCreateVPValueForSCEVExpr(), llvm::VPlan::getScalarHeader(), llvm::PredicatedScalarEvolution::getSE(), llvm::VPBlockBase::getSingleSuccessor(), llvm::VPBlockBase::getSuccessors(), llvm::PredicatedScalarEvolution::getSymbolicMaxBackedgeTakenCount(), llvm::ScalarEvolution::getTripCountFromExitCount(), llvm::VPBlockUtils::insertBlockAfter(), llvm::VPBlockUtils::insertOnEdge(), llvm::isa(), llvm::VPBasicBlock::phis(), llvm::VPlan::setTripCount(), llvm::VPBlockBase::swapSuccessors(), and llvm::zip_equal().

Referenced by llvm::VPlanTransforms::buildVPlan0().

◆ canonicalHeaderAndLatch()

bool canonicalHeaderAndLatch	(	VPBlockBase *	HeaderVPB,
		const VPDominatorTree &	VPDT )

static

Checks if HeaderVPB is a loop header block in the plain CFG; that is, it has exactly 2 predecessors (preheader and latch), where the block dominates the latch and the preheader dominates the block.

If it is a header block return true and canonicalize the predecessors of the header (making sure the preheader appears first and the latch second) and the successors of the latch (making sure the loop exit comes first). Otherwise return false.

Definition at line 382 of file VPlanConstruction.cpp.

References assert(), llvm::VPInstruction::BranchOnCond, llvm::cast(), llvm::DominatorTreeBase< NodeT, IsPostDom >::dominates(), getOpcode(), llvm::VPBlockBase::getPredecessors(), llvm::VPInstruction::Not, llvm::ArrayRef< T >::size(), std::swap(), and llvm::VPBlockBase::swapPredecessors().

Referenced by addInitialSkeleton(), and llvm::VPlanTransforms::createLoopRegions().

◆ createExtractsForLiveOuts()

void createExtractsForLiveOuts	(	VPlan &	Plan,
		VPBasicBlock *	MiddleVPBB )

static

Creates extracts for values in Plan defined in a loop region and used outside a loop region.

Definition at line 492 of file VPlanConstruction.cpp.

References assert(), B(), llvm::cast(), llvm::VPlan::getExitBlocks(), and llvm::VPBasicBlock::getFirstNonPhi().

Referenced by addInitialSkeleton().

◆ createLoopRegion()

void createLoopRegion	(	VPlan &	Plan,
		VPBlockBase *	HeaderVPB )

static

Create a new VPRegionBlock for the loop starting at HeaderVPB.

Definition at line 429 of file VPlanConstruction.cpp.

References llvm::VPBlockUtils::connectBlocks(), llvm::VPlan::createLoopRegion(), llvm::VPBlockUtils::disconnectBlocks(), llvm::VPBlockBase::getPredecessors(), llvm::VPBlockBase::setParent(), llvm::VPBlockUtils::transferSuccessors(), and llvm::vp_depth_first_shallow().

Referenced by llvm::VPlanTransforms::createLoopRegions().

◆ createWidenInductionRecipe()

VPHeaderPHIRecipe * createWidenInductionRecipe	(	PHINode *	Phi,
		VPPhi *	PhiR,
		VPIRValue *	Start,
		const InductionDescriptor &	IndDesc,
		VPlan &	Plan,
		PredicatedScalarEvolution &	PSE,
		Loop &	OrigLoop,
		DebugLoc	DL )

static

Creates a VPWidenIntOrFpInductionRecipe or VPWidenPointerInductionRecipe for Phi based on IndDesc.

Definition at line 611 of file VPlanConstruction.cpp.

References assert(), DL, llvm::VPValue::getDefiningRecipe(), llvm::vputils::getFlagsFromIndDesc(), llvm::InductionDescriptor::getKind(), llvm::VPlan::getLiveIn(), llvm::VPUser::getOperand(), llvm::vputils::getOrCreateVPValueForSCEVExpr(), llvm::ScalarEvolution::getSCEV(), llvm::vputils::getSCEVExprForVPValue(), llvm::PredicatedScalarEvolution::getSE(), llvm::InductionDescriptor::getStartValue(), llvm::InductionDescriptor::getStep(), llvm::Value::getType(), llvm::VPlan::getVF(), llvm::VPlan::getVFxUF(), llvm::InductionDescriptor::IK_FpInduction, llvm::InductionDescriptor::IK_IntInduction, llvm::InductionDescriptor::IK_PtrInduction, llvm::ScalarEvolution::isLoopInvariant(), llvm::ScalarEvolution::isSCEVable(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), llvm::PatternMatch::match(), and llvm::VPUser::setOperand().

Referenced by llvm::VPlanTransforms::createHeaderPhiRecipes().

◆ findFindIVSelect()

VPInstruction * findFindIVSelect ( VPValue * BackedgeVal )

static

Find and return the final select instruction of the FindIV result pattern for the given BackedgeVal: select(icmp ne ComputeReductionResult(ReducedIV), Sentinel), ComputeReductionResult(ReducedIV), Start.

Definition at line 1145 of file VPlanConstruction.cpp.

References llvm::cast(), llvm::dyn_cast(), llvm::vputils::findRecipe(), llvm::PatternMatch::m_Specific(), llvm::VPlanPatternMatch::m_VPValue(), and llvm::VPlanPatternMatch::matchFindIVResult().

Referenced by llvm::VPlanTransforms::handleMultiUseReductions().

◆ insertCheckBlockBeforeVectorLoop()

void insertCheckBlockBeforeVectorLoop	(	VPlan &	Plan,
		VPBasicBlock *	CheckBlockVPBB )

static

Insert CheckBlockVPBB on the edge leading to the vector preheader, connecting it to both vector and scalar preheaders.

Updates scalar preheader phis to account for the new predecessor.

Definition at line 972 of file VPlanConstruction.cpp.

References assert(), llvm::cast(), llvm::VPBlockUtils::connectBlocks(), llvm::VPlan::getScalarPreheader(), llvm::VPBlockBase::getSinglePredecessor(), llvm::VPlan::getVectorPreheader(), llvm::VPBlockUtils::insertOnEdge(), and llvm::VPBlockBase::swapSuccessors().

Referenced by llvm::VPlanTransforms::attachCheckBlock().

◆ isHeaderBB()

bool isHeaderBB	(	BasicBlock *	BB,
		Loop *	L )

static

Definition at line 96 of file VPlanConstruction.cpp.

◆ printAfterInitialConstruction()

void printAfterInitialConstruction ( VPlan & )

static

To make RUN_VPLAN_PASS print initial VPlan.

Definition at line 593 of file VPlanConstruction.cpp.

Referenced by llvm::VPlanTransforms::buildVPlan0().

◆ simplifyLiveInsWithSCEV()

void simplifyLiveInsWithSCEV	(	VPlan &	Plan,
		PredicatedScalarEvolution &	PSE )

static

Check Plan's live-in and replace them with constants, if they can be simplified via SCEV.

Definition at line 577 of file VPlanConstruction.cpp.

References llvm::CallingConv::C, llvm::dyn_cast(), llvm::VPlan::getLiveIns(), llvm::VPlan::getOrAddLiveIn(), llvm::vputils::getSCEVExprForVPValue(), and llvm::to_vector().

Referenced by llvm::VPlanTransforms::buildVPlan0().

Variable Documentation

◆ CheckBypassWeights

uint32_t CheckBypassWeights[] = {1, 127}

staticconstexpr

Definition at line 991 of file VPlanConstruction.cpp.

Referenced by addBypassBranch().

Macros

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ DEBUG_TYPE

Function Documentation

◆ addBypassBranch()

◆ addCanonicalIVRecipes()

◆ addInitialSkeleton()

◆ canonicalHeaderAndLatch()

◆ createExtractsForLiveOuts()

◆ createLoopRegion()

◆ createWidenInductionRecipe()

◆ findFindIVSelect()

◆ insertCheckBlockBeforeVectorLoop()

◆ isHeaderBB()

◆ printAfterInitialConstruction()

◆ simplifyLiveInsWithSCEV()

Variable Documentation

◆ CheckBypassWeights