LLVM 22.0.0git
|
This file contains implementations for different VPlan recipes. More...
#include "LoopVectorizationPlanner.h"
#include "VPlan.h"
#include "VPlanAnalysis.h"
#include "VPlanHelpers.h"
#include "VPlanPatternMatch.h"
#include "VPlanUtils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Value.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include "llvm/Transforms/Utils/LoopVersioning.h"
#include <cassert>
Go to the source code of this file.
Namespaces | |
namespace | llvm |
This is an optimization pass for GlobalISel generic memory operations. | |
Macros | |
#define | LV_NAME "loop-vectorize" |
#define | DEBUG_TYPE LV_NAME |
Typedefs | |
using | VectorParts = SmallVector< Value *, 2 > |
Functions | |
static BranchInst * | createCondBranch (Value *Cond, VPBasicBlock *VPBB, VPTransformState &State) |
Create a conditional branch using Cond branching to the successors of VPBB . | |
static InstructionCost | getCostForIntrinsics (Intrinsic::ID ID, ArrayRef< const VPValue * > Operands, const VPRecipeWithIRFlags &R, ElementCount VF, VPCostContext &Ctx) |
Compute the cost for the intrinsic ID with Operands , produced by R . | |
static Constant * | getSignedIntOrFpConstant (Type *Ty, int64_t C) |
A helper function that returns an integer or floating-point constant with value C. | |
static Type * | getGEPIndexTy (bool IsScalable, bool IsReverse, bool IsUnitStride, unsigned CurrentPart, IRBuilderBase &Builder) |
static void | scalarizeInstruction (const Instruction *Instr, VPReplicateRecipe *RepRecipe, const VPLane &Lane, VPTransformState &State) |
A helper function to scalarize a single Instruction in the innermost loop. | |
static Instruction * | createReverseEVL (IRBuilderBase &Builder, Value *Operand, Value *EVL, const Twine &Name) |
Use all-true mask for reverse rather than actual mask, as it avoids a dependence w/o affecting the result. | |
static Value * | createBitOrPointerCast (IRBuilderBase &Builder, Value *V, VectorType *DstVTy, const DataLayout &DL) |
static Value * | interleaveVectors (IRBuilderBase &Builder, ArrayRef< Value * > Vals, const Twine &Name) |
Return a vector containing interleaved elements from multiple smaller input vectors. | |
This file contains implementations for different VPlan recipes.
Definition in file VPlanRecipes.cpp.
#define DEBUG_TYPE LV_NAME |
Definition at line 47 of file VPlanRecipes.cpp.
#define LV_NAME "loop-vectorize" |
Definition at line 46 of file VPlanRecipes.cpp.
using VectorParts = SmallVector<Value *, 2> |
Definition at line 44 of file VPlanRecipes.cpp.
|
static |
Definition at line 3553 of file VPlanRecipes.cpp.
References assert(), llvm::IRBuilderBase::CreateBitOrPointerCast(), DL, llvm::VectorType::get(), llvm::Type::getIntNTy(), llvm::CastInst::isBitOrNoopPointerCastable(), llvm::Type::isFloatingPointTy(), and llvm::Type::isPointerTy().
Referenced by llvm::VPInterleaveRecipe::execute().
|
static |
Create a conditional branch using Cond
branching to the successors of VPBB
.
Note that the first successor is always forward (i.e. not created yet) while the second successor may already have been created (if it is a header block and VPBB is a latch).
Definition at line 542 of file VPlanRecipes.cpp.
References llvm::VPTransformState::Builder, llvm::VPTransformState::CFG, Cond, llvm::IRBuilderBase::CreateCondBr(), llvm::Instruction::eraseFromParent(), llvm::VPBlockBase::getSuccessors(), llvm::BasicBlock::getTerminator(), llvm::BranchInst::setSuccessor(), and llvm::VPTransformState::CFGState::VPBB2IRBB.
|
static |
Use all-true mask for reverse rather than actual mask, as it avoids a dependence w/o affecting the result.
Definition at line 3366 of file VPlanRecipes.cpp.
References llvm::IRBuilderBase::CreateIntrinsic(), llvm::IRBuilderBase::CreateVectorSplat(), llvm::IRBuilderBase::getTrue(), llvm::Value::getType(), and Name.
Referenced by llvm::VPWidenLoadEVLRecipe::execute(), and llvm::VPWidenStoreEVLRecipe::execute().
|
static |
Compute the cost for the intrinsic ID
with Operands
, produced by R
.
Definition at line 1719 of file VPlanRecipes.cpp.
References Arguments, llvm::VPCostContext::CostKind, llvm::enumerate(), llvm::TargetTransformInfo::getIntrinsicInstrCost(), llvm::InstructionCost::getInvalid(), Idx, llvm::VPTypeAnalysis::inferScalarType(), llvm::ElementCount::isVector(), Operands, llvm::SmallVectorTemplateBase< T, bool >::push_back(), RetTy, llvm::VPCostContext::TLI, llvm::toVectorizedTy(), llvm::toVectorTy(), llvm::VPCostContext::TTI, and llvm::VPCostContext::Types.
Referenced by llvm::VPWidenIntrinsicRecipe::computeCost(), and llvm::VPReplicateRecipe::computeCost().
|
static |
Definition at line 2511 of file VPlanRecipes.cpp.
References DL, llvm::BasicBlock::getDataLayout(), llvm::IRBuilderBase::GetInsertBlock(), llvm::IRBuilderBase::getInt32Ty(), and llvm::IRBuilderBase::getPtrTy().
Referenced by llvm::VPVectorEndPointerRecipe::execute(), and llvm::VPVectorPointerRecipe::execute().
A helper function that returns an integer or floating-point constant with value C.
Definition at line 2286 of file VPlanRecipes.cpp.
References llvm::CallingConv::C, llvm::ConstantInt::getSigned(), and llvm::Type::isIntegerTy().
Referenced by llvm::VPScalarIVStepsRecipe::execute().
|
static |
Return a vector containing interleaved elements from multiple smaller input vectors.
Definition at line 3585 of file VPlanRecipes.cpp.
References assert(), llvm::concatenateVectors(), llvm::createInterleaveMask(), llvm::IRBuilderBase::CreateShuffleVector(), llvm::IRBuilderBase::CreateVectorInterleave(), getType(), Name, and llvm::ArrayRef< T >::size().
Referenced by llvm::VPInterleaveRecipe::execute().
|
static |
A helper function to scalarize a single Instruction in the innermost loop.
Generates a sequence of scalar instances for lane Lane
. Uses the VPValue operands from RepRecipe
instead of Instr's
operands.
Definition at line 2956 of file VPlanRecipes.cpp.
References llvm::VPTransformState::AC, llvm::all_of(), llvm::VPIRFlags::applyFlags(), llvm::VPIRMetadata::applyMetadata(), assert(), llvm::VPTransformState::Builder, llvm::canVectorizeTy(), DL, llvm::enumerate(), llvm::VPTransformState::get(), llvm::VPRecipeBase::getDebugLoc(), llvm::VPLane::getFirstLane(), llvm::VPBlockBase::getParent(), llvm::VPRecipeBase::getParent(), llvm::VPBlockBase::getPlan(), llvm::VPIRFlags::getPredicate(), llvm::Value::getType(), llvm::VPlan::getVectorLoopRegion(), llvm::VPIRFlags::hasPredicate(), I, II, llvm::VPTypeAnalysis::inferScalarType(), llvm::IRBuilderBase::Insert(), llvm::vputils::isSingleScalar(), llvm::Value::mutateType(), llvm::VPUser::operands(), llvm::AssumptionCache::registerAssumption(), llvm::VPTransformState::set(), llvm::VPTransformState::setDebugLocFrom(), llvm::Value::setName(), llvm::User::setOperand(), and llvm::VPTransformState::TypeAnalysis.
Referenced by llvm::VPReplicateRecipe::execute().