LLVM  3.7.0
Scalarizer.cpp
Go to the documentation of this file.
1 //===--- Scalarizer.cpp - Scalarize vector operations ---------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass converts vector operations into scalar operations, in order
11 // to expose optimization opportunities on the individual scalar operations.
12 // It is mainly intended for targets that do not have vector units, but it
13 // may also be useful for revectorizing code to different vector widths.
14 //
15 //===----------------------------------------------------------------------===//
16 
17 #include "llvm/ADT/STLExtras.h"
18 #include "llvm/IR/IRBuilder.h"
19 #include "llvm/IR/InstVisitor.h"
20 #include "llvm/Pass.h"
22 #include "llvm/Transforms/Scalar.h"
24 
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "scalarizer"
28 
29 namespace {
30 // Used to store the scattered form of a vector.
31 typedef SmallVector<Value *, 8> ValueVector;
32 
33 // Used to map a vector Value to its scattered form. We use std::map
34 // because we want iterators to persist across insertion and because the
35 // values are relatively large.
36 typedef std::map<Value *, ValueVector> ScatterMap;
37 
38 // Lists Instructions that have been replaced with scalar implementations,
39 // along with a pointer to their scattered forms.
41 
42 // Provides a very limited vector-like interface for lazily accessing one
43 // component of a scattered vector or vector pointer.
44 class Scatterer {
45 public:
46  Scatterer() {}
47 
48  // Scatter V into Size components. If new instructions are needed,
49  // insert them before BBI in BB. If Cache is nonnull, use it to cache
50  // the results.
51  Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v,
52  ValueVector *cachePtr = nullptr);
53 
54  // Return component I, creating a new Value for it if necessary.
55  Value *operator[](unsigned I);
56 
57  // Return the number of components.
58  unsigned size() const { return Size; }
59 
60 private:
61  BasicBlock *BB;
63  Value *V;
64  ValueVector *CachePtr;
65  PointerType *PtrTy;
66  ValueVector Tmp;
67  unsigned Size;
68 };
69 
70 // FCmpSpliiter(FCI)(Builder, X, Y, Name) uses Builder to create an FCmp
71 // called Name that compares X and Y in the same way as FCI.
72 struct FCmpSplitter {
73  FCmpSplitter(FCmpInst &fci) : FCI(fci) {}
74  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
75  const Twine &Name) const {
76  return Builder.CreateFCmp(FCI.getPredicate(), Op0, Op1, Name);
77  }
78  FCmpInst &FCI;
79 };
80 
81 // ICmpSpliiter(ICI)(Builder, X, Y, Name) uses Builder to create an ICmp
82 // called Name that compares X and Y in the same way as ICI.
83 struct ICmpSplitter {
84  ICmpSplitter(ICmpInst &ici) : ICI(ici) {}
85  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
86  const Twine &Name) const {
87  return Builder.CreateICmp(ICI.getPredicate(), Op0, Op1, Name);
88  }
89  ICmpInst &ICI;
90 };
91 
92 // BinarySpliiter(BO)(Builder, X, Y, Name) uses Builder to create
93 // a binary operator like BO called Name with operands X and Y.
94 struct BinarySplitter {
95  BinarySplitter(BinaryOperator &bo) : BO(bo) {}
96  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
97  const Twine &Name) const {
98  return Builder.CreateBinOp(BO.getOpcode(), Op0, Op1, Name);
99  }
100  BinaryOperator &BO;
101 };
102 
103 // Information about a load or store that we're scalarizing.
104 struct VectorLayout {
105  VectorLayout() : VecTy(nullptr), ElemTy(nullptr), VecAlign(0), ElemSize(0) {}
106 
107  // Return the alignment of element I.
108  uint64_t getElemAlign(unsigned I) {
109  return MinAlign(VecAlign, I * ElemSize);
110  }
111 
112  // The type of the vector.
113  VectorType *VecTy;
114 
115  // The type of each element.
116  Type *ElemTy;
117 
118  // The alignment of the vector.
119  uint64_t VecAlign;
120 
121  // The size of each element.
122  uint64_t ElemSize;
123 };
124 
125 class Scalarizer : public FunctionPass,
126  public InstVisitor<Scalarizer, bool> {
127 public:
128  static char ID;
129 
130  Scalarizer() :
131  FunctionPass(ID) {
133  }
134 
135  bool doInitialization(Module &M) override;
136  bool runOnFunction(Function &F) override;
137 
138  // InstVisitor methods. They return true if the instruction was scalarized,
139  // false if nothing changed.
140  bool visitInstruction(Instruction &) { return false; }
141  bool visitSelectInst(SelectInst &SI);
142  bool visitICmpInst(ICmpInst &);
143  bool visitFCmpInst(FCmpInst &);
144  bool visitBinaryOperator(BinaryOperator &);
145  bool visitGetElementPtrInst(GetElementPtrInst &);
146  bool visitCastInst(CastInst &);
147  bool visitBitCastInst(BitCastInst &);
148  bool visitShuffleVectorInst(ShuffleVectorInst &);
149  bool visitPHINode(PHINode &);
150  bool visitLoadInst(LoadInst &);
151  bool visitStoreInst(StoreInst &);
152 
153  static void registerOptions() {
154  // This is disabled by default because having separate loads and stores
155  // makes it more likely that the -combiner-alias-analysis limits will be
156  // reached.
158  &Scalarizer::ScalarizeLoadStore>(
159  "scalarize-load-store",
160  "Allow the scalarizer pass to scalarize loads and store", false);
161  }
162 
163 private:
164  Scatterer scatter(Instruction *, Value *);
165  void gather(Instruction *, const ValueVector &);
166  bool canTransferMetadata(unsigned Kind);
167  void transferMetadata(Instruction *, const ValueVector &);
168  bool getVectorLayout(Type *, unsigned, VectorLayout &, const DataLayout &);
169  bool finish();
170 
171  template<typename T> bool splitBinary(Instruction &, const T &);
172 
173  ScatterMap Scattered;
174  GatherList Gathered;
175  unsigned ParallelLoopAccessMDKind;
176  bool ScalarizeLoadStore;
177 };
178 
179 char Scalarizer::ID = 0;
180 } // end anonymous namespace
181 
182 INITIALIZE_PASS_WITH_OPTIONS(Scalarizer, "scalarizer",
183  "Scalarize vector operations", false, false)
184 
185 Scatterer::Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v,
186  ValueVector *cachePtr)
187  : BB(bb), BBI(bbi), V(v), CachePtr(cachePtr) {
188  Type *Ty = V->getType();
189  PtrTy = dyn_cast<PointerType>(Ty);
190  if (PtrTy)
191  Ty = PtrTy->getElementType();
192  Size = Ty->getVectorNumElements();
193  if (!CachePtr)
194  Tmp.resize(Size, nullptr);
195  else if (CachePtr->empty())
196  CachePtr->resize(Size, nullptr);
197  else
198  assert(Size == CachePtr->size() && "Inconsistent vector sizes");
199 }
200 
201 // Return component I, creating a new Value for it if necessary.
202 Value *Scatterer::operator[](unsigned I) {
203  ValueVector &CV = (CachePtr ? *CachePtr : Tmp);
204  // Try to reuse a previous value.
205  if (CV[I])
206  return CV[I];
207  IRBuilder<> Builder(BB, BBI);
208  if (PtrTy) {
209  if (!CV[0]) {
210  Type *Ty =
211  PointerType::get(PtrTy->getElementType()->getVectorElementType(),
212  PtrTy->getAddressSpace());
213  CV[0] = Builder.CreateBitCast(V, Ty, V->getName() + ".i0");
214  }
215  if (I != 0)
216  CV[I] = Builder.CreateConstGEP1_32(nullptr, CV[0], I,
217  V->getName() + ".i" + Twine(I));
218  } else {
219  // Search through a chain of InsertElementInsts looking for element I.
220  // Record other elements in the cache. The new V is still suitable
221  // for all uncached indices.
222  for (;;) {
224  if (!Insert)
225  break;
226  ConstantInt *Idx = dyn_cast<ConstantInt>(Insert->getOperand(2));
227  if (!Idx)
228  break;
229  unsigned J = Idx->getZExtValue();
230  V = Insert->getOperand(0);
231  if (I == J) {
232  CV[J] = Insert->getOperand(1);
233  return CV[J];
234  } else if (!CV[J]) {
235  // Only cache the first entry we find for each index we're not actively
236  // searching for. This prevents us from going too far up the chain and
237  // caching incorrect entries.
238  CV[J] = Insert->getOperand(1);
239  }
240  }
241  CV[I] = Builder.CreateExtractElement(V, Builder.getInt32(I),
242  V->getName() + ".i" + Twine(I));
243  }
244  return CV[I];
245 }
246 
247 bool Scalarizer::doInitialization(Module &M) {
248  ParallelLoopAccessMDKind =
249  M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
250  ScalarizeLoadStore =
251  M.getContext().getOption<bool, Scalarizer, &Scalarizer::ScalarizeLoadStore>();
252  return false;
253 }
254 
255 bool Scalarizer::runOnFunction(Function &F) {
256  for (Function::iterator BBI = F.begin(), BBE = F.end(); BBI != BBE; ++BBI) {
257  BasicBlock *BB = BBI;
258  for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
259  Instruction *I = II;
260  bool Done = visit(I);
261  ++II;
262  if (Done && I->getType()->isVoidTy())
263  I->eraseFromParent();
264  }
265  }
266  return finish();
267 }
268 
269 // Return a scattered form of V that can be accessed by Point. V must be a
270 // vector or a pointer to a vector.
271 Scatterer Scalarizer::scatter(Instruction *Point, Value *V) {
272  if (Argument *VArg = dyn_cast<Argument>(V)) {
273  // Put the scattered form of arguments in the entry block,
274  // so that it can be used everywhere.
275  Function *F = VArg->getParent();
276  BasicBlock *BB = &F->getEntryBlock();
277  return Scatterer(BB, BB->begin(), V, &Scattered[V]);
278  }
279  if (Instruction *VOp = dyn_cast<Instruction>(V)) {
280  // Put the scattered form of an instruction directly after the
281  // instruction.
282  BasicBlock *BB = VOp->getParent();
283  return Scatterer(BB, std::next(BasicBlock::iterator(VOp)),
284  V, &Scattered[V]);
285  }
286  // In the fallback case, just put the scattered before Point and
287  // keep the result local to Point.
288  return Scatterer(Point->getParent(), Point, V);
289 }
290 
291 // Replace Op with the gathered form of the components in CV. Defer the
292 // deletion of Op and creation of the gathered form to the end of the pass,
293 // so that we can avoid creating the gathered form if all uses of Op are
294 // replaced with uses of CV.
295 void Scalarizer::gather(Instruction *Op, const ValueVector &CV) {
296  // Since we're not deleting Op yet, stub out its operands, so that it
297  // doesn't make anything live unnecessarily.
298  for (unsigned I = 0, E = Op->getNumOperands(); I != E; ++I)
299  Op->setOperand(I, UndefValue::get(Op->getOperand(I)->getType()));
300 
301  transferMetadata(Op, CV);
302 
303  // If we already have a scattered form of Op (created from ExtractElements
304  // of Op itself), replace them with the new form.
305  ValueVector &SV = Scattered[Op];
306  if (!SV.empty()) {
307  for (unsigned I = 0, E = SV.size(); I != E; ++I) {
308  Instruction *Old = cast<Instruction>(SV[I]);
309  CV[I]->takeName(Old);
310  Old->replaceAllUsesWith(CV[I]);
311  Old->eraseFromParent();
312  }
313  }
314  SV = CV;
315  Gathered.push_back(GatherList::value_type(Op, &SV));
316 }
317 
318 // Return true if it is safe to transfer the given metadata tag from
319 // vector to scalar instructions.
320 bool Scalarizer::canTransferMetadata(unsigned Tag) {
321  return (Tag == LLVMContext::MD_tbaa
322  || Tag == LLVMContext::MD_fpmath
326  || Tag == LLVMContext::MD_noalias
327  || Tag == ParallelLoopAccessMDKind);
328 }
329 
330 // Transfer metadata from Op to the instructions in CV if it is known
331 // to be safe to do so.
332 void Scalarizer::transferMetadata(Instruction *Op, const ValueVector &CV) {
335  for (unsigned I = 0, E = CV.size(); I != E; ++I) {
336  if (Instruction *New = dyn_cast<Instruction>(CV[I])) {
337  for (SmallVectorImpl<std::pair<unsigned, MDNode *>>::iterator
338  MI = MDs.begin(),
339  ME = MDs.end();
340  MI != ME; ++MI)
341  if (canTransferMetadata(MI->first))
342  New->setMetadata(MI->first, MI->second);
343  New->setDebugLoc(Op->getDebugLoc());
344  }
345  }
346 }
347 
348 // Try to fill in Layout from Ty, returning true on success. Alignment is
349 // the alignment of the vector, or 0 if the ABI default should be used.
350 bool Scalarizer::getVectorLayout(Type *Ty, unsigned Alignment,
351  VectorLayout &Layout, const DataLayout &DL) {
352  // Make sure we're dealing with a vector.
353  Layout.VecTy = dyn_cast<VectorType>(Ty);
354  if (!Layout.VecTy)
355  return false;
356 
357  // Check that we're dealing with full-byte elements.
358  Layout.ElemTy = Layout.VecTy->getElementType();
359  if (DL.getTypeSizeInBits(Layout.ElemTy) !=
360  DL.getTypeStoreSizeInBits(Layout.ElemTy))
361  return false;
362 
363  if (Alignment)
364  Layout.VecAlign = Alignment;
365  else
366  Layout.VecAlign = DL.getABITypeAlignment(Layout.VecTy);
367  Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy);
368  return true;
369 }
370 
371 // Scalarize two-operand instruction I, using Split(Builder, X, Y, Name)
372 // to create an instruction like I with operands X and Y and name Name.
373 template<typename Splitter>
374 bool Scalarizer::splitBinary(Instruction &I, const Splitter &Split) {
376  if (!VT)
377  return false;
378 
379  unsigned NumElems = VT->getNumElements();
380  IRBuilder<> Builder(I.getParent(), &I);
381  Scatterer Op0 = scatter(&I, I.getOperand(0));
382  Scatterer Op1 = scatter(&I, I.getOperand(1));
383  assert(Op0.size() == NumElems && "Mismatched binary operation");
384  assert(Op1.size() == NumElems && "Mismatched binary operation");
385  ValueVector Res;
386  Res.resize(NumElems);
387  for (unsigned Elem = 0; Elem < NumElems; ++Elem)
388  Res[Elem] = Split(Builder, Op0[Elem], Op1[Elem],
389  I.getName() + ".i" + Twine(Elem));
390  gather(&I, Res);
391  return true;
392 }
393 
394 bool Scalarizer::visitSelectInst(SelectInst &SI) {
395  VectorType *VT = dyn_cast<VectorType>(SI.getType());
396  if (!VT)
397  return false;
398 
399  unsigned NumElems = VT->getNumElements();
400  IRBuilder<> Builder(SI.getParent(), &SI);
401  Scatterer Op1 = scatter(&SI, SI.getOperand(1));
402  Scatterer Op2 = scatter(&SI, SI.getOperand(2));
403  assert(Op1.size() == NumElems && "Mismatched select");
404  assert(Op2.size() == NumElems && "Mismatched select");
405  ValueVector Res;
406  Res.resize(NumElems);
407 
408  if (SI.getOperand(0)->getType()->isVectorTy()) {
409  Scatterer Op0 = scatter(&SI, SI.getOperand(0));
410  assert(Op0.size() == NumElems && "Mismatched select");
411  for (unsigned I = 0; I < NumElems; ++I)
412  Res[I] = Builder.CreateSelect(Op0[I], Op1[I], Op2[I],
413  SI.getName() + ".i" + Twine(I));
414  } else {
415  Value *Op0 = SI.getOperand(0);
416  for (unsigned I = 0; I < NumElems; ++I)
417  Res[I] = Builder.CreateSelect(Op0, Op1[I], Op2[I],
418  SI.getName() + ".i" + Twine(I));
419  }
420  gather(&SI, Res);
421  return true;
422 }
423 
424 bool Scalarizer::visitICmpInst(ICmpInst &ICI) {
425  return splitBinary(ICI, ICmpSplitter(ICI));
426 }
427 
428 bool Scalarizer::visitFCmpInst(FCmpInst &FCI) {
429  return splitBinary(FCI, FCmpSplitter(FCI));
430 }
431 
432 bool Scalarizer::visitBinaryOperator(BinaryOperator &BO) {
433  return splitBinary(BO, BinarySplitter(BO));
434 }
435 
436 bool Scalarizer::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
437  VectorType *VT = dyn_cast<VectorType>(GEPI.getType());
438  if (!VT)
439  return false;
440 
441  IRBuilder<> Builder(GEPI.getParent(), &GEPI);
442  unsigned NumElems = VT->getNumElements();
443  unsigned NumIndices = GEPI.getNumIndices();
444 
445  Scatterer Base = scatter(&GEPI, GEPI.getOperand(0));
446 
448  Ops.resize(NumIndices);
449  for (unsigned I = 0; I < NumIndices; ++I)
450  Ops[I] = scatter(&GEPI, GEPI.getOperand(I + 1));
451 
452  ValueVector Res;
453  Res.resize(NumElems);
454  for (unsigned I = 0; I < NumElems; ++I) {
455  SmallVector<Value *, 8> Indices;
456  Indices.resize(NumIndices);
457  for (unsigned J = 0; J < NumIndices; ++J)
458  Indices[J] = Ops[J][I];
459  Res[I] = Builder.CreateGEP(GEPI.getSourceElementType(), Base[I], Indices,
460  GEPI.getName() + ".i" + Twine(I));
461  if (GEPI.isInBounds())
462  if (GetElementPtrInst *NewGEPI = dyn_cast<GetElementPtrInst>(Res[I]))
463  NewGEPI->setIsInBounds();
464  }
465  gather(&GEPI, Res);
466  return true;
467 }
468 
469 bool Scalarizer::visitCastInst(CastInst &CI) {
471  if (!VT)
472  return false;
473 
474  unsigned NumElems = VT->getNumElements();
475  IRBuilder<> Builder(CI.getParent(), &CI);
476  Scatterer Op0 = scatter(&CI, CI.getOperand(0));
477  assert(Op0.size() == NumElems && "Mismatched cast");
478  ValueVector Res;
479  Res.resize(NumElems);
480  for (unsigned I = 0; I < NumElems; ++I)
481  Res[I] = Builder.CreateCast(CI.getOpcode(), Op0[I], VT->getElementType(),
482  CI.getName() + ".i" + Twine(I));
483  gather(&CI, Res);
484  return true;
485 }
486 
487 bool Scalarizer::visitBitCastInst(BitCastInst &BCI) {
488  VectorType *DstVT = dyn_cast<VectorType>(BCI.getDestTy());
489  VectorType *SrcVT = dyn_cast<VectorType>(BCI.getSrcTy());
490  if (!DstVT || !SrcVT)
491  return false;
492 
493  unsigned DstNumElems = DstVT->getNumElements();
494  unsigned SrcNumElems = SrcVT->getNumElements();
495  IRBuilder<> Builder(BCI.getParent(), &BCI);
496  Scatterer Op0 = scatter(&BCI, BCI.getOperand(0));
497  ValueVector Res;
498  Res.resize(DstNumElems);
499 
500  if (DstNumElems == SrcNumElems) {
501  for (unsigned I = 0; I < DstNumElems; ++I)
502  Res[I] = Builder.CreateBitCast(Op0[I], DstVT->getElementType(),
503  BCI.getName() + ".i" + Twine(I));
504  } else if (DstNumElems > SrcNumElems) {
505  // <M x t1> -> <N*M x t2>. Convert each t1 to <N x t2> and copy the
506  // individual elements to the destination.
507  unsigned FanOut = DstNumElems / SrcNumElems;
508  Type *MidTy = VectorType::get(DstVT->getElementType(), FanOut);
509  unsigned ResI = 0;
510  for (unsigned Op0I = 0; Op0I < SrcNumElems; ++Op0I) {
511  Value *V = Op0[Op0I];
512  Instruction *VI;
513  // Look through any existing bitcasts before converting to <N x t2>.
514  // In the best case, the resulting conversion might be a no-op.
515  while ((VI = dyn_cast<Instruction>(V)) &&
516  VI->getOpcode() == Instruction::BitCast)
517  V = VI->getOperand(0);
518  V = Builder.CreateBitCast(V, MidTy, V->getName() + ".cast");
519  Scatterer Mid = scatter(&BCI, V);
520  for (unsigned MidI = 0; MidI < FanOut; ++MidI)
521  Res[ResI++] = Mid[MidI];
522  }
523  } else {
524  // <N*M x t1> -> <M x t2>. Convert each group of <N x t1> into a t2.
525  unsigned FanIn = SrcNumElems / DstNumElems;
526  Type *MidTy = VectorType::get(SrcVT->getElementType(), FanIn);
527  unsigned Op0I = 0;
528  for (unsigned ResI = 0; ResI < DstNumElems; ++ResI) {
529  Value *V = UndefValue::get(MidTy);
530  for (unsigned MidI = 0; MidI < FanIn; ++MidI)
531  V = Builder.CreateInsertElement(V, Op0[Op0I++], Builder.getInt32(MidI),
532  BCI.getName() + ".i" + Twine(ResI)
533  + ".upto" + Twine(MidI));
534  Res[ResI] = Builder.CreateBitCast(V, DstVT->getElementType(),
535  BCI.getName() + ".i" + Twine(ResI));
536  }
537  }
538  gather(&BCI, Res);
539  return true;
540 }
541 
542 bool Scalarizer::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
543  VectorType *VT = dyn_cast<VectorType>(SVI.getType());
544  if (!VT)
545  return false;
546 
547  unsigned NumElems = VT->getNumElements();
548  Scatterer Op0 = scatter(&SVI, SVI.getOperand(0));
549  Scatterer Op1 = scatter(&SVI, SVI.getOperand(1));
550  ValueVector Res;
551  Res.resize(NumElems);
552 
553  for (unsigned I = 0; I < NumElems; ++I) {
554  int Selector = SVI.getMaskValue(I);
555  if (Selector < 0)
556  Res[I] = UndefValue::get(VT->getElementType());
557  else if (unsigned(Selector) < Op0.size())
558  Res[I] = Op0[Selector];
559  else
560  Res[I] = Op1[Selector - Op0.size()];
561  }
562  gather(&SVI, Res);
563  return true;
564 }
565 
566 bool Scalarizer::visitPHINode(PHINode &PHI) {
567  VectorType *VT = dyn_cast<VectorType>(PHI.getType());
568  if (!VT)
569  return false;
570 
571  unsigned NumElems = VT->getNumElements();
572  IRBuilder<> Builder(PHI.getParent(), &PHI);
573  ValueVector Res;
574  Res.resize(NumElems);
575 
576  unsigned NumOps = PHI.getNumOperands();
577  for (unsigned I = 0; I < NumElems; ++I)
578  Res[I] = Builder.CreatePHI(VT->getElementType(), NumOps,
579  PHI.getName() + ".i" + Twine(I));
580 
581  for (unsigned I = 0; I < NumOps; ++I) {
582  Scatterer Op = scatter(&PHI, PHI.getIncomingValue(I));
583  BasicBlock *IncomingBlock = PHI.getIncomingBlock(I);
584  for (unsigned J = 0; J < NumElems; ++J)
585  cast<PHINode>(Res[J])->addIncoming(Op[J], IncomingBlock);
586  }
587  gather(&PHI, Res);
588  return true;
589 }
590 
591 bool Scalarizer::visitLoadInst(LoadInst &LI) {
592  if (!ScalarizeLoadStore)
593  return false;
594  if (!LI.isSimple())
595  return false;
596 
597  VectorLayout Layout;
598  if (!getVectorLayout(LI.getType(), LI.getAlignment(), Layout,
599  LI.getModule()->getDataLayout()))
600  return false;
601 
602  unsigned NumElems = Layout.VecTy->getNumElements();
603  IRBuilder<> Builder(LI.getParent(), &LI);
604  Scatterer Ptr = scatter(&LI, LI.getPointerOperand());
605  ValueVector Res;
606  Res.resize(NumElems);
607 
608  for (unsigned I = 0; I < NumElems; ++I)
609  Res[I] = Builder.CreateAlignedLoad(Ptr[I], Layout.getElemAlign(I),
610  LI.getName() + ".i" + Twine(I));
611  gather(&LI, Res);
612  return true;
613 }
614 
615 bool Scalarizer::visitStoreInst(StoreInst &SI) {
616  if (!ScalarizeLoadStore)
617  return false;
618  if (!SI.isSimple())
619  return false;
620 
621  VectorLayout Layout;
622  Value *FullValue = SI.getValueOperand();
623  if (!getVectorLayout(FullValue->getType(), SI.getAlignment(), Layout,
624  SI.getModule()->getDataLayout()))
625  return false;
626 
627  unsigned NumElems = Layout.VecTy->getNumElements();
628  IRBuilder<> Builder(SI.getParent(), &SI);
629  Scatterer Ptr = scatter(&SI, SI.getPointerOperand());
630  Scatterer Val = scatter(&SI, FullValue);
631 
632  ValueVector Stores;
633  Stores.resize(NumElems);
634  for (unsigned I = 0; I < NumElems; ++I) {
635  unsigned Align = Layout.getElemAlign(I);
636  Stores[I] = Builder.CreateAlignedStore(Val[I], Ptr[I], Align);
637  }
638  transferMetadata(&SI, Stores);
639  return true;
640 }
641 
642 // Delete the instructions that we scalarized. If a full vector result
643 // is still needed, recreate it using InsertElements.
644 bool Scalarizer::finish() {
645  if (Gathered.empty())
646  return false;
647  for (GatherList::iterator GMI = Gathered.begin(), GME = Gathered.end();
648  GMI != GME; ++GMI) {
649  Instruction *Op = GMI->first;
650  ValueVector &CV = *GMI->second;
651  if (!Op->use_empty()) {
652  // The value is still needed, so recreate it using a series of
653  // InsertElements.
654  Type *Ty = Op->getType();
655  Value *Res = UndefValue::get(Ty);
656  BasicBlock *BB = Op->getParent();
657  unsigned Count = Ty->getVectorNumElements();
658  IRBuilder<> Builder(BB, Op);
659  if (isa<PHINode>(Op))
660  Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
661  for (unsigned I = 0; I < Count; ++I)
662  Res = Builder.CreateInsertElement(Res, CV[I], Builder.getInt32(I),
663  Op->getName() + ".upto" + Twine(I));
664  Res->takeName(Op);
665  Op->replaceAllUsesWith(Res);
666  }
667  Op->eraseFromParent();
668  }
669  Gathered.clear();
670  Scattered.clear();
671  return true;
672 }
673 
675  return new Scalarizer();
676 }
static int getMaskValue(Constant *Mask, unsigned i)
getMaskValue - Return the index from the shuffle mask for the specified output result.
Value * getValueOperand()
Definition: Instructions.h:406
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
Definition: Instruction.cpp:70
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:104
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition: InstrTypes.h:649
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
LLVM Argument representation.
Definition: Argument.h:35
Base class for instruction visitors.
Definition: InstVisitor.h:81
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1442
Type * getSourceElementType() const
Definition: Instructions.h:926
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:114
iterator end()
Definition: Function.h:459
unsigned getNumOperands() const
Definition: User.h:138
bool isSimple() const
Definition: Instructions.h:401
static PointerType * get(Type *ElementType, unsigned AddressSpace)
PointerType::get - This constructs a pointer to an object of the specified type in a numbered address...
Definition: Type.cpp:738
ShuffleVectorInst - This instruction constructs a fixed permutation of two input vectors.
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:111
F(f)
LoadInst - an instruction for reading from memory.
Definition: Instructions.h:177
bool isSimple() const
Definition: Instructions.h:279
unsigned getNumIndices() const
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:923
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:188
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:231
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:1462
void push_back(NodeTy *val)
Definition: ilist.h:554
SelectInst - This class represents the LLVM 'select' instruction.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:79
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:389
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:517
void clear()
Definition: ilist.h:550
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:117
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:1449
This instruction compares its operands according to the predicate given to the constructor.
This class represents a no-op cast from one type to another.
StoreInst - an instruction for storing to memory.
Definition: Instructions.h:316
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:351
unsigned getNumElements() const
Return the number of elements in the Vector type.
Definition: DerivedTypes.h:432
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:256
iterator begin()
Definition: Function.h:457
Type * getElementType() const
Definition: DerivedTypes.h:323
bool isInBounds() const
isInBounds - Determine whether the GEP has the inbounds flag.
PointerType - Class to represent pointers.
Definition: DerivedTypes.h:449
static void registerOption(const char *ArgStr, const char *Desc, const ValT &InitValue)
Registers an option with the OptionRegistry singleton.
Definition: Options.h:96
GetElementPtrInst - an instruction for type-safe pointer arithmetic to access elements of arrays and ...
Definition: Instructions.h:830
InsertElementInst - This instruction inserts a single (scalar) element into a VectorType value...
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
Definition: Instructions.h:365
LLVM Basic Block Representation.
Definition: BasicBlock.h:65
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
void getAllMetadataOtherThanDebugLoc(SmallVectorImpl< std::pair< unsigned, MDNode * >> &MDs) const
getAllMetadataOtherThanDebugLoc - This does the same thing as getAllMetadata, except that it filters ...
Definition: Instruction.h:190
uint64_t getTypeStoreSizeInBits(Type *Ty) const
Returns the maximum number of bits that may be overwritten by storing the specified type; always a mu...
Definition: DataLayout.h:379
bool isVectorTy() const
isVectorTy - True if this is an instance of VectorType.
Definition: Type.h:226
const DebugLoc & getDebugLoc() const
getDebugLoc - Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:230
BasicBlock * getIncomingBlock(unsigned i) const
getIncomingBlock - Return incoming basic block number i.
This instruction compares its operands according to the predicate given to the constructor.
ValT getOption() const
Query for a debug option's value.
Definition: LLVMContext.h:176
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:294
unsigned getMDKindID(StringRef Name) const
getMDKindID - Return a unique non-zero ID for the specified metadata kind.
Value * getOperand(unsigned i) const
Definition: User.h:118
Value * getPointerOperand()
Definition: Instructions.h:284
Type * getSrcTy() const
Return the source type, as a convenience.
Definition: InstrTypes.h:654
static UndefValue * get(Type *T)
get() - Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1473
LoadInst * CreateAlignedLoad(Value *Ptr, unsigned Align, const char *Name)
Definition: IRBuilder.h:991
INITIALIZE_PASS_WITH_OPTIONS(Scalarizer,"scalarizer","Scalarize vector operations", false, false) Scatterer
Definition: Scalarizer.cpp:182
unsigned getABITypeAlignment(Type *Ty) const
Returns the minimum ABI-required alignment for the specified type.
Definition: DataLayout.cpp:674
This is the shared class of boolean and integer constants.
Definition: Constants.h:47
Value * getIncomingValue(unsigned i) const
getIncomingValue - Return incoming value number x
unsigned getVectorNumElements() const
Definition: Type.cpp:212
iterator end()
Definition: BasicBlock.h:233
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:57
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1253
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
void initializeScalarizerPass(PassRegistry &)
Type * getDestTy() const
Return the destination type, as a convenience.
Definition: InstrTypes.h:656
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:266
SequentialType * getType() const
Definition: Instructions.h:922
const BasicBlock & getEntryBlock() const
Definition: Function.h:442
static cl::opt< AlignMode > Align(cl::desc("Load/store alignment support"), cl::Hidden, cl::init(NoStrictAlign), cl::values(clEnumValN(StrictAlign,"aarch64-strict-align","Disallow all unaligned memory accesses"), clEnumValN(NoStrictAlign,"aarch64-no-strict-align","Allow unaligned memory accesses"), clEnumValEnd))
void setOperand(unsigned i, Value *Val)
Definition: User.h:122
VectorType - Class to represent vector types.
Definition: DerivedTypes.h:362
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="")
Definition: IRBuilder.h:1482
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
uint64_t MinAlign(uint64_t A, uint64_t B)
MinAlign - A and B are either alignments or offsets.
Definition: MathExtras.h:552
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:372
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
Definition: Instructions.h:243
#define I(x, y, z)
Definition: MD5.cpp:54
void size_t size
VectorType * getType() const
getType - Overload to return most specific vector type.
uint64_t getTypeStoreSize(Type *Ty) const
Returns the maximum number of bytes that may be overwritten by storing the specified type...
Definition: DataLayout.h:371
bool use_empty() const
Definition: Value.h:275
const ARM::ArchExtKind Kind
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:365
LLVM Value Representation.
Definition: Value.h:69
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:112
static VectorType * get(Type *ElementType, unsigned NumElements)
VectorType::get - This static method is the primary way to construct an VectorType.
Definition: Type.cpp:713
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:507
FunctionPass * createScalarizerPass()
Definition: Scalarizer.cpp:674
iterator getFirstInsertionPt()
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:194
static void Split(std::vector< std::string > &V, StringRef S)
Split - Splits a string of comma separated items in to a vector of strings.
Value * getPointerOperand()
Definition: Instructions.h:409
const BasicBlock * getParent() const
Definition: Instruction.h:72
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:265
bool isVoidTy() const
isVoidTy - Return true if this is 'void'.
Definition: Type.h:137
void resize(size_type N)
Definition: SmallVector.h:376