LLVM  15.0.0git
Scalarizer.cpp
Go to the documentation of this file.
1 //===- Scalarizer.cpp - Scalarize vector operations -----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass converts vector operations into scalar operations, in order
10 // to expose optimization opportunities on the individual scalar operations.
11 // It is mainly intended for targets that do not have vector units, but it
12 // may also be useful for revectorizing code to different vector widths.
13 //
14 //===----------------------------------------------------------------------===//
15 
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/Twine.h"
21 #include "llvm/IR/Argument.h"
22 #include "llvm/IR/BasicBlock.h"
23 #include "llvm/IR/Constants.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/IR/DerivedTypes.h"
26 #include "llvm/IR/Dominators.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/IR/IRBuilder.h"
29 #include "llvm/IR/InstVisitor.h"
30 #include "llvm/IR/InstrTypes.h"
31 #include "llvm/IR/Instruction.h"
32 #include "llvm/IR/Instructions.h"
33 #include "llvm/IR/Intrinsics.h"
34 #include "llvm/IR/LLVMContext.h"
35 #include "llvm/IR/Module.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/IR/Value.h"
38 #include "llvm/InitializePasses.h"
39 #include "llvm/Pass.h"
40 #include "llvm/Support/Casting.h"
43 #include <cassert>
44 #include <cstdint>
45 #include <iterator>
46 #include <map>
47 #include <utility>
48 
49 using namespace llvm;
50 
51 #define DEBUG_TYPE "scalarizer"
52 
54  "scalarize-variable-insert-extract", cl::init(true), cl::Hidden,
55  cl::desc("Allow the scalarizer pass to scalarize "
56  "insertelement/extractelement with variable index"));
57 
58 // This is disabled by default because having separate loads and stores
59 // makes it more likely that the -combiner-alias-analysis limits will be
60 // reached.
62  "scalarize-load-store", cl::init(false), cl::Hidden,
63  cl::desc("Allow the scalarizer pass to scalarize loads and store"));
64 
65 namespace {
66 
67 BasicBlock::iterator skipPastPhiNodesAndDbg(BasicBlock::iterator Itr) {
68  BasicBlock *BB = Itr->getParent();
69  if (isa<PHINode>(Itr))
70  Itr = BB->getFirstInsertionPt();
71  if (Itr != BB->end())
72  Itr = skipDebugIntrinsics(Itr);
73  return Itr;
74 }
75 
76 // Used to store the scattered form of a vector.
77 using ValueVector = SmallVector<Value *, 8>;
78 
79 // Used to map a vector Value to its scattered form. We use std::map
80 // because we want iterators to persist across insertion and because the
81 // values are relatively large.
82 using ScatterMap = std::map<Value *, ValueVector>;
83 
84 // Lists Instructions that have been replaced with scalar implementations,
85 // along with a pointer to their scattered forms.
87 
88 // Provides a very limited vector-like interface for lazily accessing one
89 // component of a scattered vector or vector pointer.
90 class Scatterer {
91 public:
92  Scatterer() = default;
93 
94  // Scatter V into Size components. If new instructions are needed,
95  // insert them before BBI in BB. If Cache is nonnull, use it to cache
96  // the results.
97  Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v, Type *PtrElemTy,
98  ValueVector *cachePtr = nullptr);
99 
100  // Return component I, creating a new Value for it if necessary.
101  Value *operator[](unsigned I);
102 
103  // Return the number of components.
104  unsigned size() const { return Size; }
105 
106 private:
107  BasicBlock *BB;
109  Value *V;
110  Type *PtrElemTy;
111  ValueVector *CachePtr;
112  ValueVector Tmp;
113  unsigned Size;
114 };
115 
116 // FCmpSpliiter(FCI)(Builder, X, Y, Name) uses Builder to create an FCmp
117 // called Name that compares X and Y in the same way as FCI.
118 struct FCmpSplitter {
119  FCmpSplitter(FCmpInst &fci) : FCI(fci) {}
120 
121  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
122  const Twine &Name) const {
123  return Builder.CreateFCmp(FCI.getPredicate(), Op0, Op1, Name);
124  }
125 
126  FCmpInst &FCI;
127 };
128 
129 // ICmpSpliiter(ICI)(Builder, X, Y, Name) uses Builder to create an ICmp
130 // called Name that compares X and Y in the same way as ICI.
131 struct ICmpSplitter {
132  ICmpSplitter(ICmpInst &ici) : ICI(ici) {}
133 
134  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
135  const Twine &Name) const {
136  return Builder.CreateICmp(ICI.getPredicate(), Op0, Op1, Name);
137  }
138 
139  ICmpInst &ICI;
140 };
141 
142 // UnarySpliiter(UO)(Builder, X, Name) uses Builder to create
143 // a unary operator like UO called Name with operand X.
144 struct UnarySplitter {
145  UnarySplitter(UnaryOperator &uo) : UO(uo) {}
146 
147  Value *operator()(IRBuilder<> &Builder, Value *Op, const Twine &Name) const {
148  return Builder.CreateUnOp(UO.getOpcode(), Op, Name);
149  }
150 
151  UnaryOperator &UO;
152 };
153 
154 // BinarySpliiter(BO)(Builder, X, Y, Name) uses Builder to create
155 // a binary operator like BO called Name with operands X and Y.
156 struct BinarySplitter {
157  BinarySplitter(BinaryOperator &bo) : BO(bo) {}
158 
159  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
160  const Twine &Name) const {
161  return Builder.CreateBinOp(BO.getOpcode(), Op0, Op1, Name);
162  }
163 
164  BinaryOperator &BO;
165 };
166 
167 // Information about a load or store that we're scalarizing.
168 struct VectorLayout {
169  VectorLayout() = default;
170 
171  // Return the alignment of element I.
172  Align getElemAlign(unsigned I) {
173  return commonAlignment(VecAlign, I * ElemSize);
174  }
175 
176  // The type of the vector.
177  VectorType *VecTy = nullptr;
178 
179  // The type of each element.
180  Type *ElemTy = nullptr;
181 
182  // The alignment of the vector.
183  Align VecAlign;
184 
185  // The size of each element.
186  uint64_t ElemSize = 0;
187 };
188 
189 template <typename T>
190 T getWithDefaultOverride(const cl::opt<T> &ClOption,
191  const llvm::Optional<T> &DefaultOverride) {
192  return ClOption.getNumOccurrences() ? ClOption
193  : DefaultOverride.getValueOr(ClOption);
194 }
195 
196 class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
197 public:
198  ScalarizerVisitor(unsigned ParallelLoopAccessMDKind, DominatorTree *DT,
200  : ParallelLoopAccessMDKind(ParallelLoopAccessMDKind), DT(DT),
201  ScalarizeVariableInsertExtract(
202  getWithDefaultOverride(ClScalarizeVariableInsertExtract,
203  Options.ScalarizeVariableInsertExtract)),
204  ScalarizeLoadStore(getWithDefaultOverride(ClScalarizeLoadStore,
205  Options.ScalarizeLoadStore)) {
206  }
207 
208  bool visit(Function &F);
209 
210  // InstVisitor methods. They return true if the instruction was scalarized,
211  // false if nothing changed.
212  bool visitInstruction(Instruction &I) { return false; }
213  bool visitSelectInst(SelectInst &SI);
214  bool visitICmpInst(ICmpInst &ICI);
215  bool visitFCmpInst(FCmpInst &FCI);
216  bool visitUnaryOperator(UnaryOperator &UO);
217  bool visitBinaryOperator(BinaryOperator &BO);
218  bool visitGetElementPtrInst(GetElementPtrInst &GEPI);
219  bool visitCastInst(CastInst &CI);
220  bool visitBitCastInst(BitCastInst &BCI);
221  bool visitInsertElementInst(InsertElementInst &IEI);
222  bool visitExtractElementInst(ExtractElementInst &EEI);
223  bool visitShuffleVectorInst(ShuffleVectorInst &SVI);
224  bool visitPHINode(PHINode &PHI);
225  bool visitLoadInst(LoadInst &LI);
226  bool visitStoreInst(StoreInst &SI);
227  bool visitCallInst(CallInst &ICI);
228 
229 private:
230  Scatterer scatter(Instruction *Point, Value *V, Type *PtrElemTy = nullptr);
231  void gather(Instruction *Op, const ValueVector &CV);
232  bool canTransferMetadata(unsigned Kind);
233  void transferMetadataAndIRFlags(Instruction *Op, const ValueVector &CV);
234  Optional<VectorLayout> getVectorLayout(Type *Ty, Align Alignment,
235  const DataLayout &DL);
236  bool finish();
237 
238  template<typename T> bool splitUnary(Instruction &, const T &);
239  template<typename T> bool splitBinary(Instruction &, const T &);
240 
241  bool splitCall(CallInst &CI);
242 
243  ScatterMap Scattered;
244  GatherList Gathered;
245 
246  SmallVector<WeakTrackingVH, 32> PotentiallyDeadInstrs;
247 
248  unsigned ParallelLoopAccessMDKind;
249 
250  DominatorTree *DT;
251 
252  const bool ScalarizeVariableInsertExtract;
253  const bool ScalarizeLoadStore;
254 };
255 
256 class ScalarizerLegacyPass : public FunctionPass {
257 public:
258  static char ID;
259 
260  ScalarizerLegacyPass() : FunctionPass(ID) {
262  }
263 
264  bool runOnFunction(Function &F) override;
265 
266  void getAnalysisUsage(AnalysisUsage& AU) const override {
269  }
270 };
271 
272 } // end anonymous namespace
273 
274 char ScalarizerLegacyPass::ID = 0;
275 INITIALIZE_PASS_BEGIN(ScalarizerLegacyPass, "scalarizer",
276  "Scalarize vector operations", false, false)
278 INITIALIZE_PASS_END(ScalarizerLegacyPass, "scalarizer",
279  "Scalarize vector operations", false, false)
280 
281 Scatterer::Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v,
282  Type *PtrElemTy, ValueVector *cachePtr)
283  : BB(bb), BBI(bbi), V(v), PtrElemTy(PtrElemTy), CachePtr(cachePtr) {
284  Type *Ty = V->getType();
285  if (Ty->isPointerTy()) {
286  assert(cast<PointerType>(Ty)->isOpaqueOrPointeeTypeMatches(PtrElemTy) &&
287  "Pointer element type mismatch");
288  Ty = PtrElemTy;
289  }
290  Size = cast<FixedVectorType>(Ty)->getNumElements();
291  if (!CachePtr)
292  Tmp.resize(Size, nullptr);
293  else if (CachePtr->empty())
294  CachePtr->resize(Size, nullptr);
295  else
296  assert(Size == CachePtr->size() && "Inconsistent vector sizes");
297 }
298 
299 // Return component I, creating a new Value for it if necessary.
300 Value *Scatterer::operator[](unsigned I) {
301  ValueVector &CV = (CachePtr ? *CachePtr : Tmp);
302  // Try to reuse a previous value.
303  if (CV[I])
304  return CV[I];
305  IRBuilder<> Builder(BB, BBI);
306  if (PtrElemTy) {
307  Type *VectorElemTy = cast<VectorType>(PtrElemTy)->getElementType();
308  if (!CV[0]) {
309  Type *NewPtrTy = PointerType::get(
310  VectorElemTy, V->getType()->getPointerAddressSpace());
311  CV[0] = Builder.CreateBitCast(V, NewPtrTy, V->getName() + ".i0");
312  }
313  if (I != 0)
314  CV[I] = Builder.CreateConstGEP1_32(VectorElemTy, CV[0], I,
315  V->getName() + ".i" + Twine(I));
316  } else {
317  // Search through a chain of InsertElementInsts looking for element I.
318  // Record other elements in the cache. The new V is still suitable
319  // for all uncached indices.
320  while (true) {
321  InsertElementInst *Insert = dyn_cast<InsertElementInst>(V);
322  if (!Insert)
323  break;
324  ConstantInt *Idx = dyn_cast<ConstantInt>(Insert->getOperand(2));
325  if (!Idx)
326  break;
327  unsigned J = Idx->getZExtValue();
328  V = Insert->getOperand(0);
329  if (I == J) {
330  CV[J] = Insert->getOperand(1);
331  return CV[J];
332  } else if (!CV[J]) {
333  // Only cache the first entry we find for each index we're not actively
334  // searching for. This prevents us from going too far up the chain and
335  // caching incorrect entries.
336  CV[J] = Insert->getOperand(1);
337  }
338  }
339  CV[I] = Builder.CreateExtractElement(V, Builder.getInt32(I),
340  V->getName() + ".i" + Twine(I));
341  }
342  return CV[I];
343 }
344 
346  if (skipFunction(F))
347  return false;
348 
349  Module &M = *F.getParent();
350  unsigned ParallelLoopAccessMDKind =
351  M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
352  DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
353  ScalarizerVisitor Impl(ParallelLoopAccessMDKind, DT, ScalarizerPassOptions());
354  return Impl.visit(F);
355 }
356 
358  return new ScalarizerLegacyPass();
359 }
360 
361 bool ScalarizerVisitor::visit(Function &F) {
362  assert(Gathered.empty() && Scattered.empty());
363 
364  // To ensure we replace gathered components correctly we need to do an ordered
365  // traversal of the basic blocks in the function.
366  ReversePostOrderTraversal<BasicBlock *> RPOT(&F.getEntryBlock());
367  for (BasicBlock *BB : RPOT) {
368  for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
369  Instruction *I = &*II;
370  bool Done = InstVisitor::visit(I);
371  ++II;
372  if (Done && I->getType()->isVoidTy())
373  I->eraseFromParent();
374  }
375  }
376  return finish();
377 }
378 
379 // Return a scattered form of V that can be accessed by Point. V must be a
380 // vector or a pointer to a vector.
381 Scatterer ScalarizerVisitor::scatter(Instruction *Point, Value *V,
382  Type *PtrElemTy) {
383  if (Argument *VArg = dyn_cast<Argument>(V)) {
384  // Put the scattered form of arguments in the entry block,
385  // so that it can be used everywhere.
386  Function *F = VArg->getParent();
387  BasicBlock *BB = &F->getEntryBlock();
388  return Scatterer(BB, BB->begin(), V, PtrElemTy, &Scattered[V]);
389  }
390  if (Instruction *VOp = dyn_cast<Instruction>(V)) {
391  // When scalarizing PHI nodes we might try to examine/rewrite InsertElement
392  // nodes in predecessors. If those predecessors are unreachable from entry,
393  // then the IR in those blocks could have unexpected properties resulting in
394  // infinite loops in Scatterer::operator[]. By simply treating values
395  // originating from instructions in unreachable blocks as undef we do not
396  // need to analyse them further.
397  if (!DT->isReachableFromEntry(VOp->getParent()))
398  return Scatterer(Point->getParent(), Point->getIterator(),
399  UndefValue::get(V->getType()), PtrElemTy);
400  // Put the scattered form of an instruction directly after the
401  // instruction, skipping over PHI nodes and debug intrinsics.
402  BasicBlock *BB = VOp->getParent();
403  return Scatterer(
404  BB, skipPastPhiNodesAndDbg(std::next(BasicBlock::iterator(VOp))), V,
405  PtrElemTy, &Scattered[V]);
406  }
407  // In the fallback case, just put the scattered before Point and
408  // keep the result local to Point.
409  return Scatterer(Point->getParent(), Point->getIterator(), V, PtrElemTy);
410 }
411 
412 // Replace Op with the gathered form of the components in CV. Defer the
413 // deletion of Op and creation of the gathered form to the end of the pass,
414 // so that we can avoid creating the gathered form if all uses of Op are
415 // replaced with uses of CV.
416 void ScalarizerVisitor::gather(Instruction *Op, const ValueVector &CV) {
417  transferMetadataAndIRFlags(Op, CV);
418 
419  // If we already have a scattered form of Op (created from ExtractElements
420  // of Op itself), replace them with the new form.
421  ValueVector &SV = Scattered[Op];
422  if (!SV.empty()) {
423  for (unsigned I = 0, E = SV.size(); I != E; ++I) {
424  Value *V = SV[I];
425  if (V == nullptr || SV[I] == CV[I])
426  continue;
427 
428  Instruction *Old = cast<Instruction>(V);
429  if (isa<Instruction>(CV[I]))
430  CV[I]->takeName(Old);
431  Old->replaceAllUsesWith(CV[I]);
432  PotentiallyDeadInstrs.emplace_back(Old);
433  }
434  }
435  SV = CV;
436  Gathered.push_back(GatherList::value_type(Op, &SV));
437 }
438 
439 // Return true if it is safe to transfer the given metadata tag from
440 // vector to scalar instructions.
441 bool ScalarizerVisitor::canTransferMetadata(unsigned Tag) {
442  return (Tag == LLVMContext::MD_tbaa
443  || Tag == LLVMContext::MD_fpmath
444  || Tag == LLVMContext::MD_tbaa_struct
445  || Tag == LLVMContext::MD_invariant_load
446  || Tag == LLVMContext::MD_alias_scope
447  || Tag == LLVMContext::MD_noalias
448  || Tag == ParallelLoopAccessMDKind
449  || Tag == LLVMContext::MD_access_group);
450 }
451 
452 // Transfer metadata from Op to the instructions in CV if it is known
453 // to be safe to do so.
454 void ScalarizerVisitor::transferMetadataAndIRFlags(Instruction *Op,
455  const ValueVector &CV) {
457  Op->getAllMetadataOtherThanDebugLoc(MDs);
458  for (unsigned I = 0, E = CV.size(); I != E; ++I) {
459  if (Instruction *New = dyn_cast<Instruction>(CV[I])) {
460  for (const auto &MD : MDs)
461  if (canTransferMetadata(MD.first))
462  New->setMetadata(MD.first, MD.second);
463  New->copyIRFlags(Op);
464  if (Op->getDebugLoc() && !New->getDebugLoc())
465  New->setDebugLoc(Op->getDebugLoc());
466  }
467  }
468 }
469 
470 // Try to fill in Layout from Ty, returning true on success. Alignment is
471 // the alignment of the vector, or None if the ABI default should be used.
473 ScalarizerVisitor::getVectorLayout(Type *Ty, Align Alignment,
474  const DataLayout &DL) {
475  VectorLayout Layout;
476  // Make sure we're dealing with a vector.
477  Layout.VecTy = dyn_cast<VectorType>(Ty);
478  if (!Layout.VecTy)
479  return None;
480  // Check that we're dealing with full-byte elements.
481  Layout.ElemTy = Layout.VecTy->getElementType();
482  if (!DL.typeSizeEqualsStoreSize(Layout.ElemTy))
483  return None;
484  Layout.VecAlign = Alignment;
485  Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy);
486  return Layout;
487 }
488 
489 // Scalarize one-operand instruction I, using Split(Builder, X, Name)
490 // to create an instruction like I with operand X and name Name.
491 template<typename Splitter>
492 bool ScalarizerVisitor::splitUnary(Instruction &I, const Splitter &Split) {
493  VectorType *VT = dyn_cast<VectorType>(I.getType());
494  if (!VT)
495  return false;
496 
497  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
499  Scatterer Op = scatter(&I, I.getOperand(0));
500  assert(Op.size() == NumElems && "Mismatched unary operation");
501  ValueVector Res;
502  Res.resize(NumElems);
503  for (unsigned Elem = 0; Elem < NumElems; ++Elem)
504  Res[Elem] = Split(Builder, Op[Elem], I.getName() + ".i" + Twine(Elem));
505  gather(&I, Res);
506  return true;
507 }
508 
509 // Scalarize two-operand instruction I, using Split(Builder, X, Y, Name)
510 // to create an instruction like I with operands X and Y and name Name.
511 template<typename Splitter>
512 bool ScalarizerVisitor::splitBinary(Instruction &I, const Splitter &Split) {
513  VectorType *VT = dyn_cast<VectorType>(I.getType());
514  if (!VT)
515  return false;
516 
517  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
519  Scatterer VOp0 = scatter(&I, I.getOperand(0));
520  Scatterer VOp1 = scatter(&I, I.getOperand(1));
521  assert(VOp0.size() == NumElems && "Mismatched binary operation");
522  assert(VOp1.size() == NumElems && "Mismatched binary operation");
523  ValueVector Res;
524  Res.resize(NumElems);
525  for (unsigned Elem = 0; Elem < NumElems; ++Elem) {
526  Value *Op0 = VOp0[Elem];
527  Value *Op1 = VOp1[Elem];
528  Res[Elem] = Split(Builder, Op0, Op1, I.getName() + ".i" + Twine(Elem));
529  }
530  gather(&I, Res);
531  return true;
532 }
533 
535  return isTriviallyVectorizable(ID);
536 }
537 
538 // All of the current scalarizable intrinsics only have one mangled type.
541  ArrayRef<Type*> Tys) {
542  return Intrinsic::getDeclaration(M, ID, Tys);
543 }
544 
545 /// If a call to a vector typed intrinsic function, split into a scalar call per
546 /// element if possible for the intrinsic.
547 bool ScalarizerVisitor::splitCall(CallInst &CI) {
548  VectorType *VT = dyn_cast<VectorType>(CI.getType());
549  if (!VT)
550  return false;
551 
552  Function *F = CI.getCalledFunction();
553  if (!F)
554  return false;
555 
556  Intrinsic::ID ID = F->getIntrinsicID();
558  return false;
559 
560  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
561  unsigned NumArgs = CI.arg_size();
562 
563  ValueVector ScalarOperands(NumArgs);
564  SmallVector<Scatterer, 8> Scattered(NumArgs);
565 
566  Scattered.resize(NumArgs);
567 
569  Tys.push_back(VT->getScalarType());
570 
571  // Assumes that any vector type has the same number of elements as the return
572  // vector type, which is true for all current intrinsics.
573  for (unsigned I = 0; I != NumArgs; ++I) {
574  Value *OpI = CI.getOperand(I);
575  if (OpI->getType()->isVectorTy()) {
576  Scattered[I] = scatter(&CI, OpI);
577  assert(Scattered[I].size() == NumElems && "mismatched call operands");
579  Tys.push_back(OpI->getType()->getScalarType());
580  } else {
581  ScalarOperands[I] = OpI;
583  Tys.push_back(OpI->getType());
584  }
585  }
586 
587  ValueVector Res(NumElems);
588  ValueVector ScalarCallOps(NumArgs);
589 
590  Function *NewIntrin = getScalarIntrinsicDeclaration(F->getParent(), ID, Tys);
591  IRBuilder<> Builder(&CI);
592 
593  // Perform actual scalarization, taking care to preserve any scalar operands.
594  for (unsigned Elem = 0; Elem < NumElems; ++Elem) {
595  ScalarCallOps.clear();
596 
597  for (unsigned J = 0; J != NumArgs; ++J) {
599  ScalarCallOps.push_back(ScalarOperands[J]);
600  else
601  ScalarCallOps.push_back(Scattered[J][Elem]);
602  }
603 
604  Res[Elem] = Builder.CreateCall(NewIntrin, ScalarCallOps,
605  CI.getName() + ".i" + Twine(Elem));
606  }
607 
608  gather(&CI, Res);
609  return true;
610 }
611 
612 bool ScalarizerVisitor::visitSelectInst(SelectInst &SI) {
613  VectorType *VT = dyn_cast<VectorType>(SI.getType());
614  if (!VT)
615  return false;
616 
617  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
619  Scatterer VOp1 = scatter(&SI, SI.getOperand(1));
620  Scatterer VOp2 = scatter(&SI, SI.getOperand(2));
621  assert(VOp1.size() == NumElems && "Mismatched select");
622  assert(VOp2.size() == NumElems && "Mismatched select");
623  ValueVector Res;
624  Res.resize(NumElems);
625 
626  if (SI.getOperand(0)->getType()->isVectorTy()) {
627  Scatterer VOp0 = scatter(&SI, SI.getOperand(0));
628  assert(VOp0.size() == NumElems && "Mismatched select");
629  for (unsigned I = 0; I < NumElems; ++I) {
630  Value *Op0 = VOp0[I];
631  Value *Op1 = VOp1[I];
632  Value *Op2 = VOp2[I];
633  Res[I] = Builder.CreateSelect(Op0, Op1, Op2,
634  SI.getName() + ".i" + Twine(I));
635  }
636  } else {
637  Value *Op0 = SI.getOperand(0);
638  for (unsigned I = 0; I < NumElems; ++I) {
639  Value *Op1 = VOp1[I];
640  Value *Op2 = VOp2[I];
641  Res[I] = Builder.CreateSelect(Op0, Op1, Op2,
642  SI.getName() + ".i" + Twine(I));
643  }
644  }
645  gather(&SI, Res);
646  return true;
647 }
648 
649 bool ScalarizerVisitor::visitICmpInst(ICmpInst &ICI) {
650  return splitBinary(ICI, ICmpSplitter(ICI));
651 }
652 
653 bool ScalarizerVisitor::visitFCmpInst(FCmpInst &FCI) {
654  return splitBinary(FCI, FCmpSplitter(FCI));
655 }
656 
657 bool ScalarizerVisitor::visitUnaryOperator(UnaryOperator &UO) {
658  return splitUnary(UO, UnarySplitter(UO));
659 }
660 
661 bool ScalarizerVisitor::visitBinaryOperator(BinaryOperator &BO) {
662  return splitBinary(BO, BinarySplitter(BO));
663 }
664 
665 bool ScalarizerVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
666  VectorType *VT = dyn_cast<VectorType>(GEPI.getType());
667  if (!VT)
668  return false;
669 
670  IRBuilder<> Builder(&GEPI);
671  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
672  unsigned NumIndices = GEPI.getNumIndices();
673 
674  // The base pointer might be scalar even if it's a vector GEP. In those cases,
675  // splat the pointer into a vector value, and scatter that vector.
676  Value *Op0 = GEPI.getOperand(0);
677  if (!Op0->getType()->isVectorTy())
678  Op0 = Builder.CreateVectorSplat(NumElems, Op0);
679  Scatterer Base = scatter(&GEPI, Op0);
680 
682  Ops.resize(NumIndices);
683  for (unsigned I = 0; I < NumIndices; ++I) {
684  Value *Op = GEPI.getOperand(I + 1);
685 
686  // The indices might be scalars even if it's a vector GEP. In those cases,
687  // splat the scalar into a vector value, and scatter that vector.
688  if (!Op->getType()->isVectorTy())
689  Op = Builder.CreateVectorSplat(NumElems, Op);
690 
691  Ops[I] = scatter(&GEPI, Op);
692  }
693 
694  ValueVector Res;
695  Res.resize(NumElems);
696  for (unsigned I = 0; I < NumElems; ++I) {
697  SmallVector<Value *, 8> Indices;
698  Indices.resize(NumIndices);
699  for (unsigned J = 0; J < NumIndices; ++J)
700  Indices[J] = Ops[J][I];
701  Res[I] = Builder.CreateGEP(GEPI.getSourceElementType(), Base[I], Indices,
702  GEPI.getName() + ".i" + Twine(I));
703  if (GEPI.isInBounds())
704  if (GetElementPtrInst *NewGEPI = dyn_cast<GetElementPtrInst>(Res[I]))
705  NewGEPI->setIsInBounds();
706  }
707  gather(&GEPI, Res);
708  return true;
709 }
710 
711 bool ScalarizerVisitor::visitCastInst(CastInst &CI) {
712  VectorType *VT = dyn_cast<VectorType>(CI.getDestTy());
713  if (!VT)
714  return false;
715 
716  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
717  IRBuilder<> Builder(&CI);
718  Scatterer Op0 = scatter(&CI, CI.getOperand(0));
719  assert(Op0.size() == NumElems && "Mismatched cast");
720  ValueVector Res;
721  Res.resize(NumElems);
722  for (unsigned I = 0; I < NumElems; ++I)
723  Res[I] = Builder.CreateCast(CI.getOpcode(), Op0[I], VT->getElementType(),
724  CI.getName() + ".i" + Twine(I));
725  gather(&CI, Res);
726  return true;
727 }
728 
729 bool ScalarizerVisitor::visitBitCastInst(BitCastInst &BCI) {
730  VectorType *DstVT = dyn_cast<VectorType>(BCI.getDestTy());
731  VectorType *SrcVT = dyn_cast<VectorType>(BCI.getSrcTy());
732  if (!DstVT || !SrcVT)
733  return false;
734 
735  unsigned DstNumElems = cast<FixedVectorType>(DstVT)->getNumElements();
736  unsigned SrcNumElems = cast<FixedVectorType>(SrcVT)->getNumElements();
737  IRBuilder<> Builder(&BCI);
738  Scatterer Op0 = scatter(&BCI, BCI.getOperand(0));
739  ValueVector Res;
740  Res.resize(DstNumElems);
741 
742  if (DstNumElems == SrcNumElems) {
743  for (unsigned I = 0; I < DstNumElems; ++I)
744  Res[I] = Builder.CreateBitCast(Op0[I], DstVT->getElementType(),
745  BCI.getName() + ".i" + Twine(I));
746  } else if (DstNumElems > SrcNumElems) {
747  // <M x t1> -> <N*M x t2>. Convert each t1 to <N x t2> and copy the
748  // individual elements to the destination.
749  unsigned FanOut = DstNumElems / SrcNumElems;
750  auto *MidTy = FixedVectorType::get(DstVT->getElementType(), FanOut);
751  unsigned ResI = 0;
752  for (unsigned Op0I = 0; Op0I < SrcNumElems; ++Op0I) {
753  Value *V = Op0[Op0I];
754  Instruction *VI;
755  // Look through any existing bitcasts before converting to <N x t2>.
756  // In the best case, the resulting conversion might be a no-op.
757  while ((VI = dyn_cast<Instruction>(V)) &&
758  VI->getOpcode() == Instruction::BitCast)
759  V = VI->getOperand(0);
760  V = Builder.CreateBitCast(V, MidTy, V->getName() + ".cast");
761  Scatterer Mid = scatter(&BCI, V);
762  for (unsigned MidI = 0; MidI < FanOut; ++MidI)
763  Res[ResI++] = Mid[MidI];
764  }
765  } else {
766  // <N*M x t1> -> <M x t2>. Convert each group of <N x t1> into a t2.
767  unsigned FanIn = SrcNumElems / DstNumElems;
768  auto *MidTy = FixedVectorType::get(SrcVT->getElementType(), FanIn);
769  unsigned Op0I = 0;
770  for (unsigned ResI = 0; ResI < DstNumElems; ++ResI) {
771  Value *V = PoisonValue::get(MidTy);
772  for (unsigned MidI = 0; MidI < FanIn; ++MidI)
773  V = Builder.CreateInsertElement(V, Op0[Op0I++], Builder.getInt32(MidI),
774  BCI.getName() + ".i" + Twine(ResI)
775  + ".upto" + Twine(MidI));
776  Res[ResI] = Builder.CreateBitCast(V, DstVT->getElementType(),
777  BCI.getName() + ".i" + Twine(ResI));
778  }
779  }
780  gather(&BCI, Res);
781  return true;
782 }
783 
784 bool ScalarizerVisitor::visitInsertElementInst(InsertElementInst &IEI) {
785  VectorType *VT = dyn_cast<VectorType>(IEI.getType());
786  if (!VT)
787  return false;
788 
789  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
790  IRBuilder<> Builder(&IEI);
791  Scatterer Op0 = scatter(&IEI, IEI.getOperand(0));
792  Value *NewElt = IEI.getOperand(1);
793  Value *InsIdx = IEI.getOperand(2);
794 
795  ValueVector Res;
796  Res.resize(NumElems);
797 
798  if (auto *CI = dyn_cast<ConstantInt>(InsIdx)) {
799  for (unsigned I = 0; I < NumElems; ++I)
800  Res[I] = CI->getValue().getZExtValue() == I ? NewElt : Op0[I];
801  } else {
802  if (!ScalarizeVariableInsertExtract)
803  return false;
804 
805  for (unsigned I = 0; I < NumElems; ++I) {
806  Value *ShouldReplace =
807  Builder.CreateICmpEQ(InsIdx, ConstantInt::get(InsIdx->getType(), I),
808  InsIdx->getName() + ".is." + Twine(I));
809  Value *OldElt = Op0[I];
810  Res[I] = Builder.CreateSelect(ShouldReplace, NewElt, OldElt,
811  IEI.getName() + ".i" + Twine(I));
812  }
813  }
814 
815  gather(&IEI, Res);
816  return true;
817 }
818 
819 bool ScalarizerVisitor::visitExtractElementInst(ExtractElementInst &EEI) {
820  VectorType *VT = dyn_cast<VectorType>(EEI.getOperand(0)->getType());
821  if (!VT)
822  return false;
823 
824  unsigned NumSrcElems = cast<FixedVectorType>(VT)->getNumElements();
825  IRBuilder<> Builder(&EEI);
826  Scatterer Op0 = scatter(&EEI, EEI.getOperand(0));
827  Value *ExtIdx = EEI.getOperand(1);
828 
829  if (auto *CI = dyn_cast<ConstantInt>(ExtIdx)) {
830  Value *Res = Op0[CI->getValue().getZExtValue()];
831  gather(&EEI, {Res});
832  return true;
833  }
834 
835  if (!ScalarizeVariableInsertExtract)
836  return false;
837 
838  Value *Res = UndefValue::get(VT->getElementType());
839  for (unsigned I = 0; I < NumSrcElems; ++I) {
840  Value *ShouldExtract =
841  Builder.CreateICmpEQ(ExtIdx, ConstantInt::get(ExtIdx->getType(), I),
842  ExtIdx->getName() + ".is." + Twine(I));
843  Value *Elt = Op0[I];
844  Res = Builder.CreateSelect(ShouldExtract, Elt, Res,
845  EEI.getName() + ".upto" + Twine(I));
846  }
847  gather(&EEI, {Res});
848  return true;
849 }
850 
851 bool ScalarizerVisitor::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
852  VectorType *VT = dyn_cast<VectorType>(SVI.getType());
853  if (!VT)
854  return false;
855 
856  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
857  Scatterer Op0 = scatter(&SVI, SVI.getOperand(0));
858  Scatterer Op1 = scatter(&SVI, SVI.getOperand(1));
859  ValueVector Res;
860  Res.resize(NumElems);
861 
862  for (unsigned I = 0; I < NumElems; ++I) {
863  int Selector = SVI.getMaskValue(I);
864  if (Selector < 0)
865  Res[I] = UndefValue::get(VT->getElementType());
866  else if (unsigned(Selector) < Op0.size())
867  Res[I] = Op0[Selector];
868  else
869  Res[I] = Op1[Selector - Op0.size()];
870  }
871  gather(&SVI, Res);
872  return true;
873 }
874 
875 bool ScalarizerVisitor::visitPHINode(PHINode &PHI) {
876  VectorType *VT = dyn_cast<VectorType>(PHI.getType());
877  if (!VT)
878  return false;
879 
880  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
881  IRBuilder<> Builder(&PHI);
882  ValueVector Res;
883  Res.resize(NumElems);
884 
885  unsigned NumOps = PHI.getNumOperands();
886  for (unsigned I = 0; I < NumElems; ++I)
887  Res[I] = Builder.CreatePHI(VT->getElementType(), NumOps,
888  PHI.getName() + ".i" + Twine(I));
889 
890  for (unsigned I = 0; I < NumOps; ++I) {
891  Scatterer Op = scatter(&PHI, PHI.getIncomingValue(I));
892  BasicBlock *IncomingBlock = PHI.getIncomingBlock(I);
893  for (unsigned J = 0; J < NumElems; ++J)
894  cast<PHINode>(Res[J])->addIncoming(Op[J], IncomingBlock);
895  }
896  gather(&PHI, Res);
897  return true;
898 }
899 
900 bool ScalarizerVisitor::visitLoadInst(LoadInst &LI) {
901  if (!ScalarizeLoadStore)
902  return false;
903  if (!LI.isSimple())
904  return false;
905 
906  Optional<VectorLayout> Layout = getVectorLayout(
907  LI.getType(), LI.getAlign(), LI.getModule()->getDataLayout());
908  if (!Layout)
909  return false;
910 
911  unsigned NumElems = cast<FixedVectorType>(Layout->VecTy)->getNumElements();
912  IRBuilder<> Builder(&LI);
913  Scatterer Ptr = scatter(&LI, LI.getPointerOperand(), LI.getType());
914  ValueVector Res;
915  Res.resize(NumElems);
916 
917  for (unsigned I = 0; I < NumElems; ++I)
918  Res[I] = Builder.CreateAlignedLoad(Layout->VecTy->getElementType(), Ptr[I],
919  Align(Layout->getElemAlign(I)),
920  LI.getName() + ".i" + Twine(I));
921  gather(&LI, Res);
922  return true;
923 }
924 
925 bool ScalarizerVisitor::visitStoreInst(StoreInst &SI) {
926  if (!ScalarizeLoadStore)
927  return false;
928  if (!SI.isSimple())
929  return false;
930 
931  Value *FullValue = SI.getValueOperand();
932  Optional<VectorLayout> Layout = getVectorLayout(
933  FullValue->getType(), SI.getAlign(), SI.getModule()->getDataLayout());
934  if (!Layout)
935  return false;
936 
937  unsigned NumElems = cast<FixedVectorType>(Layout->VecTy)->getNumElements();
939  Scatterer VPtr = scatter(&SI, SI.getPointerOperand(), FullValue->getType());
940  Scatterer VVal = scatter(&SI, FullValue);
941 
942  ValueVector Stores;
943  Stores.resize(NumElems);
944  for (unsigned I = 0; I < NumElems; ++I) {
945  Value *Val = VVal[I];
946  Value *Ptr = VPtr[I];
947  Stores[I] = Builder.CreateAlignedStore(Val, Ptr, Layout->getElemAlign(I));
948  }
949  transferMetadataAndIRFlags(&SI, Stores);
950  return true;
951 }
952 
953 bool ScalarizerVisitor::visitCallInst(CallInst &CI) {
954  return splitCall(CI);
955 }
956 
957 // Delete the instructions that we scalarized. If a full vector result
958 // is still needed, recreate it using InsertElements.
959 bool ScalarizerVisitor::finish() {
960  // The presence of data in Gathered or Scattered indicates changes
961  // made to the Function.
962  if (Gathered.empty() && Scattered.empty())
963  return false;
964  for (const auto &GMI : Gathered) {
965  Instruction *Op = GMI.first;
966  ValueVector &CV = *GMI.second;
967  if (!Op->use_empty()) {
968  // The value is still needed, so recreate it using a series of
969  // InsertElements.
970  Value *Res = PoisonValue::get(Op->getType());
971  if (auto *Ty = dyn_cast<VectorType>(Op->getType())) {
972  BasicBlock *BB = Op->getParent();
973  unsigned Count = cast<FixedVectorType>(Ty)->getNumElements();
975  if (isa<PHINode>(Op))
976  Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
977  for (unsigned I = 0; I < Count; ++I)
978  Res = Builder.CreateInsertElement(Res, CV[I], Builder.getInt32(I),
979  Op->getName() + ".upto" + Twine(I));
980  Res->takeName(Op);
981  } else {
982  assert(CV.size() == 1 && Op->getType() == CV[0]->getType());
983  Res = CV[0];
984  if (Op == Res)
985  continue;
986  }
987  Op->replaceAllUsesWith(Res);
988  }
989  PotentiallyDeadInstrs.emplace_back(Op);
990  }
991  Gathered.clear();
992  Scattered.clear();
993 
995 
996  return true;
997 }
998 
1000  Module &M = *F.getParent();
1001  unsigned ParallelLoopAccessMDKind =
1002  M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
1004  ScalarizerVisitor Impl(ParallelLoopAccessMDKind, DT, Options);
1005  bool Changed = Impl.visit(F);
1006  PreservedAnalyses PA;
1008  return Changed ? PA : PreservedAnalyses::all();
1009 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:76
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:28
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:65
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1410
Insert
Vector Rotate Left Mask Mask Insert
Definition: README_P9.txt:112
llvm::BasicBlock::iterator
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:87
llvm::Type::isPointerTy
bool isPointerTy() const
True if this is an instance of PointerType.
Definition: Type.h:218
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:780
llvm::ExtractElementInst
This instruction extracts a single (scalar) element from a VectorType value.
Definition: Instructions.h:1872
llvm::Function
Definition: Function.h:60
Pass.h
llvm::PointerType::get
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Definition: Type.cpp:727
llvm::BitCastInst
This class represents a no-op cast from one type to another.
Definition: Instructions.h:5225
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:309
llvm::SmallVector< Value *, 8 >
llvm::IRBuilder<>
Local.h
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
llvm::isVectorIntrinsicWithScalarOpAtArg
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
Definition: VectorUtils.cpp:101
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:139
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Module.h
llvm::Optional
Definition: APInt.h:33
llvm::isVectorIntrinsicWithOverloadTypeAtArg
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, unsigned OpdIdx)
Identifies if the vector form of the intrinsic has a operand that has an overloaded type.
Definition: VectorUtils.cpp:119
llvm::GetElementPtrInst::getNumIndices
unsigned getNumIndices() const
Definition: Instructions.h:1097
T
#define T
Definition: Mips16ISelLowering.cpp:341
llvm::VectorType::getElementType
Type * getElementType() const
Definition: DerivedTypes.h:422
llvm::Intrinsic::not_intrinsic
@ not_intrinsic
Definition: Intrinsics.h:45
ClScalarizeLoadStore
static cl::opt< bool > ClScalarizeLoadStore("scalarize-load-store", cl::init(false), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize loads and store"))
llvm::LoadInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:268
llvm::UnaryOperator
Definition: InstrTypes.h:101
getScalarIntrinsicDeclaration
static Function * getScalarIntrinsicDeclaration(Module *M, Intrinsic::ID ID, ArrayRef< Type * > Tys)
Definition: Scalarizer.cpp:539
llvm::LoadInst::getAlign
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:224
llvm::CastInst::getDestTy
Type * getDestTy() const
Return the destination type, as a convenience.
Definition: InstrTypes.h:683
llvm::GetElementPtrInst::getSourceElementType
Type * getSourceElementType() const
Definition: Instructions.h:1004
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
Scalarizer.h
llvm::GetElementPtrInst::isInBounds
bool isInBounds() const
Determine whether the GEP has the inbounds flag.
Definition: Instructions.cpp:1832
Instruction.h
CommandLine.h
operations
Scalarize vector operations
Definition: Scalarizer.cpp:279
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
bb
< i1 > br i1 label label bb bb
Definition: README.txt:978
llvm::AArch64Layout::VectorLayout
VectorLayout
Definition: AArch64BaseInfo.h:569
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
Constants.h
llvm::PHINode::getIncomingValue
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
Definition: Instructions.h:2760
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::isTriviallyVectorizable
bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
Definition: VectorUtils.cpp:44
Intrinsics.h
Twine.h
InstrTypes.h
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation or the function signa...
Definition: InstrTypes.h:1396
llvm::FCmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1360
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::Optional::getValueOr
constexpr T getValueOr(U &&value) const &
Definition: Optional.h:289
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:227
llvm::InsertElementInst
This instruction inserts a single (scalar) element into a VectorType value.
Definition: Instructions.h:1936
false
Definition: StackSlotColoring.cpp:141
llvm::ShuffleVectorInst::getMaskValue
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
Definition: Instructions.h:2063
llvm::skipDebugIntrinsics
BasicBlock::iterator skipDebugIntrinsics(BasicBlock::iterator It)
Advance It while it points to a debug instruction and return the result.
Definition: BasicBlock.cpp:490
llvm::Instruction
Definition: Instruction.h:42
llvm::DominatorTreeWrapperPass
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:302
ClScalarizeVariableInsertExtract
static cl::opt< bool > ClScalarizeVariableInsertExtract("scalarize-variable-insert-extract", cl::init(true), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize " "insertelement/extractelement with variable index"))
Options
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
Definition: PassBuilderBindings.cpp:48
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:619
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1769
llvm::cl::Option::getNumOccurrences
int getNumOccurrences() const
Definition: CommandLine.h:395
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:919
llvm::ScalarizerPass::run
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Definition: Scalarizer.cpp:999
llvm::FixedVectorType::get
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:684
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::CastInst::getSrcTy
Type * getSrcTy() const
Return the source type, as a convenience.
Definition: InstrTypes.h:681
llvm::None
const NoneType None
Definition: None.h:24
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::ShuffleVectorInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:2054
Type.h
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
llvm::ARM_PROC::IE
@ IE
Definition: ARMBaseInfo.h:27
llvm::VectorType
Base class of all SIMD vector types.
Definition: DerivedTypes.h:389
VectorUtils.h
llvm::SPIRV::Decoration::Alignment
@ Alignment
BasicBlock.h
llvm::cl::opt< bool >
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:305
VI
@ VI
Definition: SIInstrInfo.cpp:7834
llvm::ScalarizerPassOptions
Definition: Scalarizer.h:28
llvm::ICmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1186
uint64_t
llvm::PreservedAnalyses::preserve
void preserve()
Mark an analysis as preserved.
Definition: PassManager.h:173
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::GetElementPtrInst
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:929
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
llvm::InstVisitor::visit
void visit(Iterator Start, Iterator End)
Definition: InstVisitor.h:87
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::createScalarizerPass
FunctionPass * createScalarizerPass()
Create a legacy pass manager instance of the Scalarizer pass.
Definition: Scalarizer.cpp:357
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::DominatorTree::isReachableFromEntry
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
Definition: Dominators.cpp:335
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1737
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1586
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::BinaryOperator
Definition: InstrTypes.h:188
DataLayout.h
InstVisitor.h
llvm::LoadInst::isSimple
bool isSimple() const
Definition: Instructions.h:260
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::AnalysisUsage::addPreserved
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Definition: PassAnalysisSupport.h:98
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:529
llvm::ms_demangle::IntrinsicFunctionKind::New
@ New
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::initializeScalarizerLegacyPassPass
void initializeScalarizerLegacyPassPass(PassRegistry &)
llvm::InstVisitor
Base class for instruction visitors.
Definition: InstVisitor.h:78
llvm::CastInst
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:429
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:305
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:176
Argument.h
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:142
runOnFunction
static bool runOnFunction(Function &F, bool PostInlining)
Definition: EntryExitInstrumenter.cpp:69
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::commonAlignment
Align commonAlignment(Align A, Align B)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:211
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:338
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
llvm::CallBase::arg_size
unsigned arg_size() const
Definition: InstrTypes.h:1339
Casting.h
isTriviallyScalariable
static bool isTriviallyScalariable(Intrinsic::ID ID)
Definition: Scalarizer.cpp:534
Function.h
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(ScalarizerLegacyPass, "scalarizer", "Scalarize vector operations", false, false) INITIALIZE_PASS_END(ScalarizerLegacyPass
llvm::ReversePostOrderTraversal
Definition: PostOrderIterator.h:291
scalarizer
scalarizer
Definition: Scalarizer.cpp:278
llvm::DominatorTreeAnalysis
Analysis pass which computes a DominatorTree.
Definition: Dominators.h:267
llvm::InsertElementInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:1969
llvm::ShuffleVectorInst
This instruction constructs a fixed permutation of two input vectors.
Definition: Instructions.h:2008
Instructions.h
PostOrderIterator.h
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
SmallVector.h
Dominators.h
llvm::CastInst::getOpcode
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition: InstrTypes.h:676
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::PHINode::getIncomingBlock
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Definition: Instructions.h:2780
llvm::PHINode
Definition: Instructions.h:2664
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:398
DerivedTypes.h
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1474
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
LLVMContext.h
llvm::Value::takeName
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:378
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::cl::desc
Definition: CommandLine.h:405
Value.h
InitializePasses.h
llvm::RecursivelyDeleteTriviallyDeadInstructionsPermissive
bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
Definition: Local.cpp:532
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37
llvm::PoisonValue::get
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1788