InterleavedAccessPass.cpp File Reference

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/InterleavedAccess.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <utility>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "interleaved-access"

Functions
	INITIALIZE_PASS_BEGIN (InterleavedAccess, DEBUG_TYPE, "Lower interleaved memory accesses to target specific intrinsics", false, false) INITIALIZE_PASS_END(InterleavedAccess
static bool	isDeInterleaveMask (ArrayRef< int > Mask, unsigned &Factor, unsigned &Index, unsigned MaxFactor, unsigned NumLoadElements)
	Check if the mask is a DE-interleave mask for an interleaved load.
static bool	isReInterleaveMask (ShuffleVectorInst *SVI, unsigned &Factor, unsigned MaxFactor)
	Check if the mask can be used in an interleaved store.
static Value *	getMaskOperand (IntrinsicInst *II)
static std::pair< Value *, APInt >	getMask (Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
static std::pair< Value *, APInt >	getMask (Value *WideMask, unsigned Factor, VectorType *LeafValueTy)
static void	getGapMask (const Constant &MaskConst, unsigned Factor, unsigned LeafMaskLen, APInt &GapMask)

Variables
static cl::opt< bool >	LowerInterleavedAccesses ("lower-interleaved-accesses", cl::desc("Enable lowering interleaved accesses to intrinsics"), cl::init(true), cl::Hidden)
	DEBUG_TYPE
Lower interleaved memory accesses to target specific	intrinsics
Lower interleaved memory accesses to target specific	false

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "interleaved-access"

Definition at line 78 of file InterleavedAccessPass.cpp.

Function Documentation

getGapMask()

void getGapMask ( const Constant & MaskConst, unsigned Factor, unsigned LeafMaskLen, APInt & GapMask )

static

Definition at line 572 of file InterleavedAccessPass.cpp.

References assert(), llvm::CallingConv::C, llvm::APInt::clearBit(), F, llvm::Constant::getAggregateElement(), and llvm::APInt::getBitWidth().

Referenced by getMask().

getMask() [1/2]

std::pair< Value *, APInt > getMask ( Value * WideMask, unsigned Factor, ElementCount LeafValueEC )

static

Definition at line 590 of file InterleavedAccessPass.cpp.

References llvm::all_of(), llvm::CallingConv::C, llvm::cast(), llvm::dyn_cast(), llvm::enumerate(), F, llvm::ConstantVector::get(), llvm::VectorType::get(), llvm::APInt::getAllOnes(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), getGapMask(), llvm::Type::getInt1Ty(), llvm::getInterleaveIntrinsicFactor(), getMask(), llvm::ConstantVector::getSplat(), llvm::IRBuilder(), llvm::isa(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isFixed(), llvm::ShuffleVectorInst::isInterleaveMask(), and llvm::Splat.

Referenced by llvm::VPlanTransforms::addMinimumIterationCheck(), llvm::VPInterleaveRecipe::clone(), llvm::VPReplicateRecipe::clone(), llvm::VPWidenLoadRecipe::clone(), llvm::VPWidenStoreRecipe::clone(), combineConcatVectorOps(), llvm::EpilogueVectorizerMainLoop::createIterationCountCheck(), getMask(), getMask(), llvm::VPInterleaveEVLRecipe::getNumStoreOperands(), llvm::VPInterleaveRecipe::getNumStoreOperands(), lowerShuffleAsLanePermuteAndRepeatedMask(), and llvm::VPBlendRecipe::print().

getMask() [2/2]

std::pair< Value *, APInt > getMask ( Value * WideMask, unsigned Factor, VectorType * LeafValueTy )

static

Definition at line 279 of file InterleavedAccessPass.cpp.

References getMask().

getMaskOperand()

Value * getMaskOperand ( IntrinsicInst * II )

static

Definition at line 256 of file InterleavedAccessPass.cpp.

References II, and llvm_unreachable.

INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN	(	InterleavedAccess	,
		DEBUG_TYPE	,
		"Lower interleaved memory accesses to target specific intrinsics"	,
		false	,
		false	)

References DEBUG_TYPE, and INITIALIZE_PASS_DEPENDENCY.

isDeInterleaveMask()

bool isDeInterleaveMask ( ArrayRef< int > Mask, unsigned & Factor, unsigned & Index, unsigned MaxFactor, unsigned NumLoadElements )

static

Check if the mask is a DE-interleave mask for an interleaved load.

E.g. DE-interleave masks (Factor = 2) could be: <0, 2, 4, 6> (mask of index 0 to extract even elements) <1, 3, 5, 7> (mask of index 1 to extract odd elements)

Definition at line 212 of file InterleavedAccessPass.cpp.

References llvm::ShuffleVectorInst::isDeInterleaveMaskOfFactor().

isReInterleaveMask()

bool isReInterleaveMask ( ShuffleVectorInst * SVI, unsigned & Factor, unsigned MaxFactor )

static

Check if the mask can be used in an interleaved store.

It checks for a more general pattern than the RE-interleave mask. I.e. <x, y, ... z, x+1, y+1, ...z+1, x+2, y+2, ...z+2, ...> E.g. For a Factor of 2 (LaneLen=4): <4, 32, 5, 33, 6, 34, 7, 35> E.g. For a Factor of 3 (LaneLen=4): <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19> E.g. For a Factor of 4 (LaneLen=2): <8, 2, 12, 4, 9, 3, 13, 5>

The particular case of an RE-interleave mask is: I.e. <0, LaneLen, ... , LaneLen*(Factor - 1), 1, LaneLen + 1, ...> E.g. For a Factor of 2 (LaneLen=4): <0, 4, 1, 5, 2, 6, 3, 7>

Definition at line 241 of file InterleavedAccessPass.cpp.

References llvm::ShuffleVectorInst::getShuffleMask(), and llvm::ShuffleVectorInst::isInterleave().

Variable Documentation

DEBUG_TYPE

DEBUG_TYPE

Definition at line 199 of file InterleavedAccessPass.cpp.

false

Lower interleaved memory accesses to target specific false

Definition at line 200 of file InterleavedAccessPass.cpp.

intrinsics

Lower interleaved memory accesses to target specific intrinsics

Definition at line 200 of file InterleavedAccessPass.cpp.

LowerInterleavedAccesses

cl::opt< bool > LowerInterleavedAccesses("lower-interleaved-accesses", cl::desc("Enable lowering interleaved accesses to intrinsics"), cl::init(true), cl::Hidden) ( "lower-interleaved-accesses" , cl::desc("Enable lowering interleaved accesses to intrinsics") , cl::init(true) , cl::Hidden  )

static