27 #define DEBUG_TYPE "scalarizer"
36 typedef std::map<Value *, ValueVector> ScatterMap;
52 ValueVector *cachePtr =
nullptr);
55 Value *operator[](
unsigned I);
58 unsigned size()
const {
return Size; }
64 ValueVector *CachePtr;
73 FCmpSplitter(
FCmpInst &fci) : FCI(fci) {}
84 ICmpSplitter(
ICmpInst &ici) : ICI(ici) {}
94 struct BinarySplitter {
105 VectorLayout() : VecTy(
nullptr), ElemTy(
nullptr), VecAlign(0), ElemSize(0) {}
108 uint64_t getElemAlign(
unsigned I) {
109 return MinAlign(VecAlign, I * ElemSize);
135 bool doInitialization(
Module &M)
override;
136 bool runOnFunction(
Function &
F)
override;
140 bool visitInstruction(
Instruction &) {
return false; }
153 static void registerOptions() {
158 &Scalarizer::ScalarizeLoadStore>(
159 "scalarize-load-store",
160 "Allow the scalarizer pass to scalarize loads and store",
false);
166 bool canTransferMetadata(
unsigned Kind);
167 void transferMetadata(
Instruction *,
const ValueVector &);
171 template<
typename T>
bool splitBinary(
Instruction &,
const T &);
173 ScatterMap Scattered;
175 unsigned ParallelLoopAccessMDKind;
176 bool ScalarizeLoadStore;
183 "Scalarize vector operations",
false,
false)
186 ValueVector *cachePtr)
187 : BB(bb), BBI(bbi), V(v), CachePtr(cachePtr) {
188 Type *Ty = V->getType();
191 Ty = PtrTy->getElementType();
194 Tmp.resize(Size,
nullptr);
195 else if (CachePtr->empty())
196 CachePtr->resize(Size,
nullptr);
198 assert(Size == CachePtr->size() &&
"Inconsistent vector sizes");
202 Value *Scatterer::operator[](
unsigned I) {
203 ValueVector &CV = (CachePtr ? *CachePtr : Tmp);
212 PtrTy->getAddressSpace());
213 CV[0] = Builder.CreateBitCast(V, Ty, V->getName() +
".i0");
216 CV[
I] = Builder.CreateConstGEP1_32(
nullptr, CV[0], I,
217 V->getName() +
".i" +
Twine(I));
241 CV[
I] = Builder.CreateExtractElement(V, Builder.getInt32(I),
242 V->getName() +
".i" +
Twine(I));
247 bool Scalarizer::doInitialization(
Module &M) {
248 ParallelLoopAccessMDKind =
255 bool Scalarizer::runOnFunction(
Function &
F) {
260 bool Done = visit(I);
272 if (
Argument *VArg = dyn_cast<Argument>(V)) {
277 return Scatterer(BB, BB->
begin(), V, &Scattered[V]);
288 return Scatterer(Point->
getParent(), Point, V);
295 void Scalarizer::gather(
Instruction *Op,
const ValueVector &CV) {
301 transferMetadata(Op, CV);
305 ValueVector &SV = Scattered[Op];
307 for (
unsigned I = 0, E = SV.size(); I != E; ++
I) {
315 Gathered.
push_back(GatherList::value_type(Op, &SV));
320 bool Scalarizer::canTransferMetadata(
unsigned Tag) {
327 || Tag == ParallelLoopAccessMDKind);
332 void Scalarizer::transferMetadata(
Instruction *Op,
const ValueVector &CV) {
335 for (
unsigned I = 0, E = CV.size(); I != E; ++
I) {
336 if (
Instruction *New = dyn_cast<Instruction>(CV[I])) {
341 if (canTransferMetadata(
MI->first))
342 New->setMetadata(
MI->first,
MI->second);
350 bool Scalarizer::getVectorLayout(
Type *Ty,
unsigned Alignment,
358 Layout.ElemTy = Layout.VecTy->getElementType();
364 Layout.VecAlign = Alignment;
373 template<
typename Splitter>
383 assert(Op0.size() == NumElems &&
"Mismatched binary operation");
384 assert(Op1.size() == NumElems &&
"Mismatched binary operation");
386 Res.resize(NumElems);
387 for (
unsigned Elem = 0; Elem < NumElems; ++Elem)
388 Res[Elem] =
Split(Builder, Op0[Elem], Op1[Elem],
394 bool Scalarizer::visitSelectInst(
SelectInst &SI) {
401 Scatterer Op1 = scatter(&SI, SI.
getOperand(1));
402 Scatterer Op2 = scatter(&SI, SI.
getOperand(2));
403 assert(Op1.size() == NumElems &&
"Mismatched select");
404 assert(Op2.size() == NumElems &&
"Mismatched select");
406 Res.resize(NumElems);
409 Scatterer Op0 = scatter(&SI, SI.
getOperand(0));
410 assert(Op0.size() == NumElems &&
"Mismatched select");
411 for (
unsigned I = 0; I < NumElems; ++
I)
416 for (
unsigned I = 0; I < NumElems; ++
I)
417 Res[I] = Builder.CreateSelect(Op0, Op1[I], Op2[I],
424 bool Scalarizer::visitICmpInst(
ICmpInst &ICI) {
425 return splitBinary(ICI, ICmpSplitter(ICI));
428 bool Scalarizer::visitFCmpInst(
FCmpInst &FCI) {
429 return splitBinary(FCI, FCmpSplitter(FCI));
433 return splitBinary(BO, BinarySplitter(BO));
445 Scatterer Base = scatter(&GEPI, GEPI.
getOperand(0));
449 for (
unsigned I = 0; I < NumIndices; ++
I)
450 Ops[I] = scatter(&GEPI, GEPI.
getOperand(I + 1));
453 Res.resize(NumElems);
454 for (
unsigned I = 0; I < NumElems; ++
I) {
456 Indices.
resize(NumIndices);
457 for (
unsigned J = 0; J < NumIndices; ++J)
458 Indices[J] = Ops[J][I];
463 NewGEPI->setIsInBounds();
469 bool Scalarizer::visitCastInst(
CastInst &CI) {
476 Scatterer Op0 = scatter(&CI, CI.
getOperand(0));
477 assert(Op0.size() == NumElems &&
"Mismatched cast");
479 Res.resize(NumElems);
480 for (
unsigned I = 0; I < NumElems; ++
I)
487 bool Scalarizer::visitBitCastInst(
BitCastInst &BCI) {
490 if (!DstVT || !SrcVT)
494 unsigned SrcNumElems = SrcVT->getNumElements();
496 Scatterer Op0 = scatter(&BCI, BCI.
getOperand(0));
498 Res.resize(DstNumElems);
500 if (DstNumElems == SrcNumElems) {
501 for (
unsigned I = 0; I < DstNumElems; ++
I)
504 }
else if (DstNumElems > SrcNumElems) {
507 unsigned FanOut = DstNumElems / SrcNumElems;
510 for (
unsigned Op0I = 0; Op0I < SrcNumElems; ++Op0I) {
511 Value *V = Op0[Op0I];
515 while ((VI = dyn_cast<Instruction>(V)) &&
518 V = Builder.CreateBitCast(V, MidTy, V->
getName() +
".cast");
519 Scatterer Mid = scatter(&BCI, V);
520 for (
unsigned MidI = 0; MidI < FanOut; ++MidI)
521 Res[ResI++] = Mid[MidI];
525 unsigned FanIn = SrcNumElems / DstNumElems;
528 for (
unsigned ResI = 0; ResI < DstNumElems; ++ResI) {
530 for (
unsigned MidI = 0; MidI < FanIn; ++MidI)
531 V = Builder.CreateInsertElement(V, Op0[Op0I++], Builder.
getInt32(MidI),
533 +
".upto" +
Twine(MidI));
548 Scatterer Op0 = scatter(&SVI, SVI.
getOperand(0));
549 Scatterer Op1 = scatter(&SVI, SVI.
getOperand(1));
551 Res.resize(NumElems);
553 for (
unsigned I = 0; I < NumElems; ++
I) {
557 else if (
unsigned(Selector) < Op0.size())
558 Res[I] = Op0[Selector];
560 Res[
I] = Op1[Selector - Op0.size()];
574 Res.resize(NumElems);
577 for (
unsigned I = 0; I < NumElems; ++
I)
581 for (
unsigned I = 0; I < NumOps; ++
I) {
584 for (
unsigned J = 0; J < NumElems; ++J)
585 cast<PHINode>(Res[J])->addIncoming(Op[J], IncomingBlock);
591 bool Scalarizer::visitLoadInst(
LoadInst &LI) {
592 if (!ScalarizeLoadStore)
602 unsigned NumElems = Layout.VecTy->getNumElements();
606 Res.resize(NumElems);
608 for (
unsigned I = 0; I < NumElems; ++
I)
615 bool Scalarizer::visitStoreInst(
StoreInst &SI) {
616 if (!ScalarizeLoadStore)
627 unsigned NumElems = Layout.VecTy->getNumElements();
630 Scatterer Val = scatter(&SI, FullValue);
633 Stores.resize(NumElems);
634 for (
unsigned I = 0; I < NumElems; ++
I) {
635 unsigned Align = Layout.getElemAlign(I);
636 Stores[
I] = Builder.CreateAlignedStore(Val[I], Ptr[I], Align);
638 transferMetadata(&SI, Stores);
644 bool Scalarizer::finish() {
645 if (Gathered.empty())
647 for (GatherList::iterator GMI = Gathered.begin(), GME = Gathered.end();
650 ValueVector &CV = *GMI->second;
659 if (isa<PHINode>(Op))
661 for (
unsigned I = 0; I < Count; ++
I)
662 Res = Builder.CreateInsertElement(Res, CV[I], Builder.getInt32(I),
675 return new Scalarizer();
static int getMaskValue(Constant *Mask, unsigned i)
getMaskValue - Return the index from the shuffle mask for the specified output result.
Value * getValueOperand()
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
A parsed version of the target data layout string in and methods for querying it. ...
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVM Argument representation.
Base class for instruction visitors.
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Type * getSourceElementType() const
A Module instance is used to store all the information related to an LLVM module. ...
unsigned getNumOperands() const
static PointerType * get(Type *ElementType, unsigned AddressSpace)
PointerType::get - This constructs a pointer to an object of the specified type in a numbered address...
ShuffleVectorInst - This instruction constructs a fixed permutation of two input vectors.
const Function * getParent() const
Return the enclosing method, or null if none.
LoadInst - an instruction for reading from memory.
unsigned getNumIndices() const
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
StringRef getName() const
Return a constant reference to the value's name.
iterator begin()
Instruction iterator methods.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
void push_back(NodeTy *val)
SelectInst - This class represents the LLVM 'select' instruction.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
This is the base class for all instructions that perform data casts.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
This instruction compares its operands according to the predicate given to the constructor.
This class represents a no-op cast from one type to another.
StoreInst - an instruction for storing to memory.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
unsigned getNumElements() const
Return the number of elements in the Vector type.
void takeName(Value *V)
Transfer the name from V to this value.
Type * getElementType() const
bool isInBounds() const
isInBounds - Determine whether the GEP has the inbounds flag.
PointerType - Class to represent pointers.
static void registerOption(const char *ArgStr, const char *Desc, const ValT &InitValue)
Registers an option with the OptionRegistry singleton.
GetElementPtrInst - an instruction for type-safe pointer arithmetic to access elements of arrays and ...
InsertElementInst - This instruction inserts a single (scalar) element into a VectorType value...
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
void getAllMetadataOtherThanDebugLoc(SmallVectorImpl< std::pair< unsigned, MDNode * >> &MDs) const
getAllMetadataOtherThanDebugLoc - This does the same thing as getAllMetadata, except that it filters ...
uint64_t getTypeStoreSizeInBits(Type *Ty) const
Returns the maximum number of bits that may be overwritten by storing the specified type; always a mu...
bool isVectorTy() const
isVectorTy - True if this is an instance of VectorType.
const DebugLoc & getDebugLoc() const
getDebugLoc - Return the debug location for this node as a DebugLoc.
BasicBlock * getIncomingBlock(unsigned i) const
getIncomingBlock - Return incoming basic block number i.
This instruction compares its operands according to the predicate given to the constructor.
ValT getOption() const
Query for a debug option's value.
FunctionPass class - This class is used to implement most global optimizations.
unsigned getMDKindID(StringRef Name) const
getMDKindID - Return a unique non-zero ID for the specified metadata kind.
Value * getOperand(unsigned i) const
Value * getPointerOperand()
Type * getSrcTy() const
Return the source type, as a convenience.
static UndefValue * get(Type *T)
get() - Static factory methods - Return an 'undef' object of the specified type.
LoadInst * CreateAlignedLoad(Value *Ptr, unsigned Align, const char *Name)
INITIALIZE_PASS_WITH_OPTIONS(Scalarizer,"scalarizer","Scalarize vector operations", false, false) Scatterer
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
This is the shared class of boolean and integer constants.
Value * getIncomingValue(unsigned i) const
getIncomingValue - Return incoming value number x
unsigned getVectorNumElements() const
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Type * getType() const
All values are typed, get the type of this value.
void initializeScalarizerPass(PassRegistry &)
Type * getDestTy() const
Return the destination type, as a convenience.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
SequentialType * getType() const
const BasicBlock & getEntryBlock() const
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
void setOperand(unsigned i, Value *Val)
VectorType - Class to represent vector types.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="")
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
VectorType * getType() const
getType - Overload to return most specific vector type.
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
const ARM::ArchExtKind Kind
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
static VectorType * get(Type *ElementType, unsigned NumElements)
VectorType::get - This static method is the primary way to construct an VectorType.
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
FunctionPass * createScalarizerPass()
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
static void Split(std::vector< std::string > &V, StringRef S)
Split - Splits a string of comma separated items in to a vector of strings.
Value * getPointerOperand()
const BasicBlock * getParent() const
LLVMContext & getContext() const
Get the global data context.
bool isVoidTy() const
isVoidTy - Return true if this is 'void'.