VectorCombine.cpp File Reference

#include "llvm/Transforms/Vectorize/VectorCombine.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/LoopUtils.h"
#include <numeric>
#include <queue>
#include "llvm/Transforms/Utils/InstructionWorklist.h"

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "vector-combine"

Typedefs
using	InstLane = std::pair< Use *, int >

Functions
	STATISTIC (NumVecLoad, "Number of vector loads formed")
	STATISTIC (NumVecCmp, "Number of vector compares formed")
	STATISTIC (NumVecBO, "Number of vector binops formed")
	STATISTIC (NumVecCmpBO, "Number of vector compare + binop formed")
	STATISTIC (NumShufOfBitcast, "Number of shuffles moved after bitcast")
	STATISTIC (NumScalarBO, "Number of scalar binops formed")
	STATISTIC (NumScalarCmp, "Number of scalar compares formed")
static Value *	peekThroughBitcasts (Value *V)
	Return the source operand of a potentially bitcasted value.
static bool	canWidenLoad (LoadInst *Load, const TargetTransformInfo &TTI)
static Value *	createShiftShuffle (Value *Vec, unsigned OldIndex, unsigned NewIndex, IRBuilder<> &Builder)
	Create a shuffle that translates (shifts) 1 element from the input vector to a new element location.
static ExtractElementInst *	translateExtract (ExtractElementInst *ExtElt, unsigned NewIndex, IRBuilder<> &Builder)
	Given an extract element instruction with constant index operand, shuffle the source vector (shift the scalar element) to a NewIndex for extraction.
static bool	isMemModifiedBetween (BasicBlock::iterator Begin, BasicBlock::iterator End, const MemoryLocation &Loc, AAResults &AA)
static ScalarizationResult	canScalarizeAccess (VectorType *VecTy, Value *Idx, Instruction *CtxI, AssumptionCache &AC, const DominatorTree &DT)
	Check if it is legal to scalarize a memory access to VecTy at index Idx.
static Align	computeAlignmentAfterScalarization (Align VectorAlignment, Type *ScalarType, Value *Idx, const DataLayout &DL)
	The memory operation on a vector of ScalarType had alignment of VectorAlignment.
static InstLane	lookThroughShuffles (Use *U, int Lane)
static SmallVector< InstLane >	generateInstLaneVectorFromOperand (ArrayRef< InstLane > Item, int Op)
static bool	isFreeConcat (ArrayRef< InstLane > Item, TTI::TargetCostKind CostKind, const TargetTransformInfo &TTI)
	Detect concat of multiple values into a vector.
static Value *	generateNewInstTree (ArrayRef< InstLane > Item, FixedVectorType *Ty, const SmallPtrSet< Use *, 4 > &IdentityLeafs, const SmallPtrSet< Use *, 4 > &SplatLeafs, const SmallPtrSet< Use *, 4 > &ConcatLeafs, IRBuilder<> &Builder, const TargetTransformInfo *TTI)

Variables
static cl::opt< bool >	DisableVectorCombine ("disable-vector-combine", cl::init(false), cl::Hidden, cl::desc("Disable all vector combine transforms"))
static cl::opt< bool >	DisableBinopExtractShuffle ("disable-binop-extract-shuffle", cl::init(false), cl::Hidden, cl::desc("Disable binop extract to shuffle transforms"))
static cl::opt< unsigned >	MaxInstrsToScan ("vector-combine-max-scan-instrs", cl::init(30), cl::Hidden, cl::desc("Max number of instructions to scan for vector combining."))
static const unsigned	InvalidIndex = std::numeric_limits<unsigned>::max()

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "vector-combine"

Definition at line 37 of file VectorCombine.cpp.

Typedef Documentation

InstLane

using InstLane = std::pair<Use *, int>

Definition at line 2034 of file VectorCombine.cpp.

Function Documentation

canScalarizeAccess()

static ScalarizationResult canScalarizeAccess ( VectorType *  VecTy, Value *  Idx, Instruction *  CtxI, AssumptionCache &  AC, const DominatorTree &  DT  )

static

Check if it is legal to scalarize a memory access to VecTy at index Idx.

Idx must access a valid vector element.

Definition at line 1220 of file VectorCombine.cpp.

References llvm::ConstantRange::binaryAnd(), llvm::CallingConv::C, llvm::computeConstantRange(), llvm::ConstantRange::contains(), llvm::ConstantInt::getValue(), Idx, llvm::isGuaranteedNotToBePoison(), llvm::PatternMatch::m_And(), llvm::PatternMatch::m_ConstantInt(), llvm::PatternMatch::m_URem(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::match(), and llvm::ConstantRange::urem().

canWidenLoad()

static bool canWidenLoad ( LoadInst *  Load, const TargetTransformInfo &  TTI  )

static

Definition at line 157 of file VectorCombine.cpp.

References llvm::TargetTransformInfo::getMinVectorRegisterBitWidth(), llvm::Type::getPrimitiveSizeInBits(), and llvm::mustSuppressSpeculation().

computeAlignmentAfterScalarization()

static Align computeAlignmentAfterScalarization ( Align  VectorAlignment, Type *  ScalarType, Value *  Idx, const DataLayout &  DL  )

static

The memory operation on a vector of ScalarType had alignment of VectorAlignment.

Compute the maximal, but conservatively correct, alignment that will be valid for the memory operation on a single scalar element of the same type with index Idx.

Definition at line 1267 of file VectorCombine.cpp.

References llvm::CallingConv::C, llvm::commonAlignment(), DL, and Idx.

createShiftShuffle()

static Value * createShiftShuffle ( Value *  Vec, unsigned  OldIndex, unsigned  NewIndex, IRBuilder<> &  Builder  )

static

Create a shuffle that translates (shifts) 1 element from the input vector to a new element location.

Definition at line 513 of file VectorCombine.cpp.

References llvm::IRBuilderBase::CreateShuffleVector(), llvm::Value::getType(), and llvm::PoisonMaskElem.

Referenced by translateExtract().

generateInstLaneVectorFromOperand()

static SmallVector< InstLane > generateInstLaneVectorFromOperand ( ArrayRef< InstLane >  Item, int  Op  )

static

Definition at line 2055 of file VectorCombine.cpp.

References llvm::SmallVectorImpl< T >::emplace_back(), lookThroughShuffles(), and llvm::PoisonMaskElem.

Referenced by generateNewInstTree().

generateNewInstTree()

static Value * generateNewInstTree ( ArrayRef< InstLane >  Item, FixedVectorType *  Ty, const SmallPtrSet< Use *, 4 > &  IdentityLeafs, const SmallPtrSet< Use *, 4 > &  SplatLeafs, const SmallPtrSet< Use *, 4 > &  ConcatLeafs, IRBuilder<> &  Builder, const TargetTransformInfo *  TTI  )

static

Definition at line 2101 of file VectorCombine.cpp.

References assert(), llvm::SmallPtrSetImpl< PtrType >::contains(), llvm::IRBuilderBase::CreateBinOp(), llvm::IRBuilderBase::CreateCast(), llvm::IRBuilderBase::CreateCmp(), llvm::IRBuilderBase::CreateIntrinsic(), llvm::IRBuilderBase::CreateSelect(), llvm::IRBuilderBase::CreateShuffleVector(), llvm::IRBuilderBase::CreateUnOp(), llvm::ArrayRef< T >::front(), generateInstLaneVectorFromOperand(), generateNewInstTree(), llvm::FixedVectorType::get(), llvm::FixedVectorType::getNumElements(), I, Idx, II, llvm::isVectorIntrinsicWithScalarOpAtArg(), llvm::propagateIRFlags(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ArrayRef< T >::size(), and llvm::SmallVectorBase< Size_T >::size().

Referenced by generateNewInstTree().

isFreeConcat()

static bool isFreeConcat ( ArrayRef< InstLane >  Item, TTI::TargetCostKind  CostKind, const TargetTransformInfo &  TTI  )

static

Detect concat of multiple values into a vector.

Definition at line 2069 of file VectorCombine.cpp.

References llvm::SmallVectorTemplateCommon< T, typename >::begin(), CostKind, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::ArrayRef< T >::front(), llvm::TargetTransformInfo::getShuffleCost(), llvm::Value::getType(), llvm::isPowerOf2_32(), llvm::ArrayRef< T >::size(), and llvm::TargetTransformInfo::SK_PermuteTwoSrc.

isMemModifiedBetween()

static bool isMemModifiedBetween ( BasicBlock::iterator  Begin, BasicBlock::iterator  End, const MemoryLocation &  Loc, AAResults &  AA  )

static

Definition at line 1151 of file VectorCombine.cpp.

References End, llvm::AAResults::getModRefInfo(), llvm::isModSet(), and MaxInstrsToScan.

lookThroughShuffles()

static InstLane lookThroughShuffles ( Use *  U, int  Lane  )

static

Definition at line 2036 of file VectorCombine.cpp.

References llvm::PoisonMaskElem.

Referenced by generateInstLaneVectorFromOperand().

peekThroughBitcasts()

static Value * peekThroughBitcasts ( Value *  V )

static

Return the source operand of a potentially bitcasted value.

If there is no bitcast, return the input value itself.

Definition at line 151 of file VectorCombine.cpp.

STATISTIC() [1/7]

STATISTIC	(	NumScalarBO	,
		"Number of scalar binops formed"
	)

STATISTIC() [2/7]

STATISTIC	(	NumScalarCmp	,
		"Number of scalar compares formed"
	)

STATISTIC() [3/7]

STATISTIC	(	NumShufOfBitcast	,
		"Number of shuffles moved after bitcast"
	)

STATISTIC() [4/7]

STATISTIC	(	NumVecBO	,
		"Number of vector binops formed"
	)

STATISTIC() [5/7]

STATISTIC	(	NumVecCmp	,
		"Number of vector compares formed"
	)

STATISTIC() [6/7]

STATISTIC	(	NumVecCmpBO	,
		"Number of vector compare + binop formed"
	)

STATISTIC() [7/7]

STATISTIC	(	NumVecLoad	,
		"Number of vector loads formed"
	)

translateExtract()

static ExtractElementInst * translateExtract ( ExtractElementInst *  ExtElt, unsigned  NewIndex, IRBuilder<> &  Builder  )

static

Given an extract element instruction with constant index operand, shuffle the source vector (shift the scalar element) to a NewIndex for extraction.

Return null if the input can be constant folded, so that we are not creating unnecessary instructions.

Definition at line 528 of file VectorCombine.cpp.

References assert(), llvm::CallingConv::C, llvm::IRBuilderBase::CreateExtractElement(), createShiftShuffle(), llvm::ExtractElementInst::getIndexOperand(), llvm::ExtractElementInst::getVectorOperand(), and X.

Variable Documentation

DisableBinopExtractShuffle

cl::opt< bool > DisableBinopExtractShuffle("disable-binop-extract-shuffle", cl::init(false), cl::Hidden, cl::desc("Disable binop extract to shuffle transforms")) ( "disable-binop-extract-shuffle"  , cl::init(false)  , cl::Hidden  , cl::desc("Disable binop extract to shuffle transforms")    )

static

DisableVectorCombine

cl::opt< bool > DisableVectorCombine("disable-vector-combine", cl::init(false), cl::Hidden, cl::desc("Disable all vector combine transforms")) ( "disable-vector-combine"  , cl::init(false)  , cl::Hidden  , cl::desc("Disable all vector combine transforms")    )

static

InvalidIndex

const unsigned InvalidIndex = std::numeric_limits<unsigned>::max()

static

Definition at line 63 of file VectorCombine.cpp.

MaxInstrsToScan

cl::opt< unsigned > MaxInstrsToScan("vector-combine-max-scan-instrs", cl::init(30), cl::Hidden, cl::desc("Max number of instructions to scan for vector combining.")) ( "vector-combine-max-scan-instrs"  , cl::init(30)  , cl::Hidden  , cl::desc("Max number of instructions to scan for vector combining.")    )

static

Referenced by isMemModifiedBetween().