InterleavedAccessPass.cpp File Reference

#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/InterleavedAccess.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/CodeGen/TargetSubtargetInfo.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/Local.h"
#include <cassert>
#include <utility>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "interleaved-access"

Functions
	INITIALIZE_PASS_BEGIN (InterleavedAccess, DEBUG_TYPE, "Lower interleaved memory accesses to target specific intrinsics", false, false) INITIALIZE_PASS_END(InterleavedAccess
static bool	isDeInterleaveMask (ArrayRef< int > Mask, unsigned &Factor, unsigned &Index, unsigned MaxFactor, unsigned NumLoadElements)
	Check if the mask is a DE-interleave mask for an interleaved load.
static bool	isReInterleaveMask (ShuffleVectorInst *SVI, unsigned &Factor, unsigned MaxFactor)
	Check if the mask can be used in an interleaved store.
static Value *	getMaskOperand (IntrinsicInst *II)
static std::pair< Value *, APInt >	getMask (Value *WideMask, unsigned Factor, ElementCount LeafValueEC)
static std::pair< Value *, APInt >	getMask (Value *WideMask, unsigned Factor, VectorType *LeafValueTy)
static void	getGapMask (const Constant &MaskConst, unsigned Factor, unsigned LeafMaskLen, APInt &GapMask)

Variables
static cl::opt< bool >	LowerInterleavedAccesses ("lower-interleaved-accesses", cl::desc("Enable lowering interleaved accesses to intrinsics"), cl::init(true), cl::Hidden)
	DEBUG_TYPE
Lower interleaved memory accesses to target specific	intrinsics
Lower interleaved memory accesses to target specific	false

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "interleaved-access"

Definition at line 78 of file InterleavedAccessPass.cpp.

Function Documentation

getGapMask()

static void getGapMask ( const Constant &  MaskConst, unsigned  Factor, unsigned  LeafMaskLen, APInt &  GapMask  )

static

Definition at line 573 of file InterleavedAccessPass.cpp.

References assert(), llvm::CallingConv::C, llvm::APInt::clearBit(), F, llvm::Constant::getAggregateElement(), llvm::APInt::getBitWidth(), and Idx.

Referenced by getMask().

getMask() [1/2]

static std::pair< Value *, APInt > getMask ( Value *  WideMask, unsigned  Factor, ElementCount  LeafValueEC  )

static

Definition at line 591 of file InterleavedAccessPass.cpp.

References llvm::all_of(), llvm::CallingConv::C, llvm::IRBuilderBase::CreateAnd(), llvm::IRBuilderBase::CreateExtractVector(), llvm::enumerate(), F, llvm::ConstantVector::get(), llvm::VectorType::get(), llvm::APInt::getAllOnes(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), getGapMask(), llvm::Type::getInt1Ty(), llvm::getInterleaveIntrinsicFactor(), getMask(), llvm::ConstantVector::getSplat(), Idx, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::isFixed(), llvm::ShuffleVectorInst::isInterleaveMask(), and llvm::Splat.

Referenced by llvm::VPlanTransforms::addMinimumIterationCheck(), llvm::VPInterleaveRecipe::clone(), llvm::VPReplicateRecipe::clone(), llvm::VPWidenLoadRecipe::clone(), llvm::VPWidenStoreRecipe::clone(), combineConcatVectorOps(), llvm::EpilogueVectorizerMainLoop::createIterationCountCheck(), getMask(), and lowerShuffleAsLanePermuteAndRepeatedMask().

getMask() [2/2]

static std::pair< Value *, APInt > getMask ( Value *  WideMask, unsigned  Factor, VectorType *  LeafValueTy  )

static

Definition at line 279 of file InterleavedAccessPass.cpp.

References getMask().

getMaskOperand()

static Value * getMaskOperand ( IntrinsicInst *  II )

static

Definition at line 256 of file InterleavedAccessPass.cpp.

References II, and llvm_unreachable.

INITIALIZE_PASS_BEGIN()

INITIALIZE_PASS_BEGIN	(	InterleavedAccess	,
		DEBUG_TYPE	,
		"Lower interleaved memory accesses to target specific intrinsics"	,
		false	,
		false
	)

isDeInterleaveMask()

static bool isDeInterleaveMask ( ArrayRef< int >  Mask, unsigned &  Factor, unsigned &  Index, unsigned  MaxFactor, unsigned  NumLoadElements  )

static

Check if the mask is a DE-interleave mask for an interleaved load.

E.g. DE-interleave masks (Factor = 2) could be: <0, 2, 4, 6> (mask of index 0 to extract even elements) <1, 3, 5, 7> (mask of index 1 to extract odd elements)

Definition at line 212 of file InterleavedAccessPass.cpp.

References llvm::ShuffleVectorInst::isDeInterleaveMaskOfFactor().

isReInterleaveMask()

static bool isReInterleaveMask ( ShuffleVectorInst *  SVI, unsigned &  Factor, unsigned  MaxFactor  )

static

Check if the mask can be used in an interleaved store.

It checks for a more general pattern than the RE-interleave mask. I.e. <x, y, ... z, x+1, y+1, ...z+1, x+2, y+2, ...z+2, ...> E.g. For a Factor of 2 (LaneLen=4): <4, 32, 5, 33, 6, 34, 7, 35> E.g. For a Factor of 3 (LaneLen=4): <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19> E.g. For a Factor of 4 (LaneLen=2): <8, 2, 12, 4, 9, 3, 13, 5>

The particular case of an RE-interleave mask is: I.e. <0, LaneLen, ... , LaneLen*(Factor - 1), 1, LaneLen + 1, ...> E.g. For a Factor of 2 (LaneLen=4): <0, 4, 1, 5, 2, 6, 3, 7>

Definition at line 241 of file InterleavedAccessPass.cpp.

References llvm::ShuffleVectorInst::getShuffleMask(), and llvm::ShuffleVectorInst::isInterleave().

Variable Documentation

DEBUG_TYPE

DEBUG_TYPE

Definition at line 199 of file InterleavedAccessPass.cpp.

false

Lower interleaved memory accesses to target specific false

Definition at line 200 of file InterleavedAccessPass.cpp.

intrinsics

Lower interleaved memory accesses to target specific intrinsics

Definition at line 200 of file InterleavedAccessPass.cpp.

LowerInterleavedAccesses

cl::opt< bool > LowerInterleavedAccesses("lower-interleaved-accesses", cl::desc("Enable lowering interleaved accesses to intrinsics"), cl::init(true), cl::Hidden) ( "lower-interleaved-accesses"  , cl::desc("Enable lowering interleaved accesses to intrinsics")  , cl::init(true)  , cl::Hidden    )

static