LLVM  14.0.0git
Scalarizer.cpp
Go to the documentation of this file.
1 //===- Scalarizer.cpp - Scalarize vector operations -----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass converts vector operations into scalar operations, in order
10 // to expose optimization opportunities on the individual scalar operations.
11 // It is mainly intended for targets that do not have vector units, but it
12 // may also be useful for revectorizing code to different vector widths.
13 //
14 //===----------------------------------------------------------------------===//
15 
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/Twine.h"
21 #include "llvm/IR/Argument.h"
22 #include "llvm/IR/BasicBlock.h"
23 #include "llvm/IR/Constants.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/IR/DerivedTypes.h"
26 #include "llvm/IR/Dominators.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/IR/IRBuilder.h"
29 #include "llvm/IR/InstVisitor.h"
30 #include "llvm/IR/InstrTypes.h"
31 #include "llvm/IR/Instruction.h"
32 #include "llvm/IR/Instructions.h"
33 #include "llvm/IR/Intrinsics.h"
34 #include "llvm/IR/LLVMContext.h"
35 #include "llvm/IR/Module.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/IR/Value.h"
38 #include "llvm/InitializePasses.h"
39 #include "llvm/Pass.h"
40 #include "llvm/Support/Casting.h"
43 #include "llvm/Transforms/Scalar.h"
45 #include <cassert>
46 #include <cstdint>
47 #include <iterator>
48 #include <map>
49 #include <utility>
50 
51 using namespace llvm;
52 
53 #define DEBUG_TYPE "scalarizer"
54 
56  "scalarize-variable-insert-extract", cl::init(true), cl::Hidden,
57  cl::desc("Allow the scalarizer pass to scalarize "
58  "insertelement/extractelement with variable index"));
59 
60 // This is disabled by default because having separate loads and stores
61 // makes it more likely that the -combiner-alias-analysis limits will be
62 // reached.
63 static cl::opt<bool>
64  ScalarizeLoadStore("scalarize-load-store", cl::init(false), cl::Hidden,
65  cl::desc("Allow the scalarizer pass to scalarize loads and store"));
66 
67 namespace {
68 
69 BasicBlock::iterator skipPastPhiNodesAndDbg(BasicBlock::iterator Itr) {
70  BasicBlock *BB = Itr->getParent();
71  if (isa<PHINode>(Itr))
72  Itr = BB->getFirstInsertionPt();
73  if (Itr != BB->end())
74  Itr = skipDebugIntrinsics(Itr);
75  return Itr;
76 }
77 
78 // Used to store the scattered form of a vector.
79 using ValueVector = SmallVector<Value *, 8>;
80 
81 // Used to map a vector Value to its scattered form. We use std::map
82 // because we want iterators to persist across insertion and because the
83 // values are relatively large.
84 using ScatterMap = std::map<Value *, ValueVector>;
85 
86 // Lists Instructions that have been replaced with scalar implementations,
87 // along with a pointer to their scattered forms.
89 
90 // Provides a very limited vector-like interface for lazily accessing one
91 // component of a scattered vector or vector pointer.
92 class Scatterer {
93 public:
94  Scatterer() = default;
95 
96  // Scatter V into Size components. If new instructions are needed,
97  // insert them before BBI in BB. If Cache is nonnull, use it to cache
98  // the results.
99  Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v,
100  ValueVector *cachePtr = nullptr);
101 
102  // Return component I, creating a new Value for it if necessary.
103  Value *operator[](unsigned I);
104 
105  // Return the number of components.
106  unsigned size() const { return Size; }
107 
108 private:
109  BasicBlock *BB;
111  Value *V;
112  ValueVector *CachePtr;
113  PointerType *PtrTy;
114  ValueVector Tmp;
115  unsigned Size;
116 };
117 
118 // FCmpSpliiter(FCI)(Builder, X, Y, Name) uses Builder to create an FCmp
119 // called Name that compares X and Y in the same way as FCI.
120 struct FCmpSplitter {
121  FCmpSplitter(FCmpInst &fci) : FCI(fci) {}
122 
123  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
124  const Twine &Name) const {
125  return Builder.CreateFCmp(FCI.getPredicate(), Op0, Op1, Name);
126  }
127 
128  FCmpInst &FCI;
129 };
130 
131 // ICmpSpliiter(ICI)(Builder, X, Y, Name) uses Builder to create an ICmp
132 // called Name that compares X and Y in the same way as ICI.
133 struct ICmpSplitter {
134  ICmpSplitter(ICmpInst &ici) : ICI(ici) {}
135 
136  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
137  const Twine &Name) const {
138  return Builder.CreateICmp(ICI.getPredicate(), Op0, Op1, Name);
139  }
140 
141  ICmpInst &ICI;
142 };
143 
144 // UnarySpliiter(UO)(Builder, X, Name) uses Builder to create
145 // a unary operator like UO called Name with operand X.
146 struct UnarySplitter {
147  UnarySplitter(UnaryOperator &uo) : UO(uo) {}
148 
149  Value *operator()(IRBuilder<> &Builder, Value *Op, const Twine &Name) const {
150  return Builder.CreateUnOp(UO.getOpcode(), Op, Name);
151  }
152 
153  UnaryOperator &UO;
154 };
155 
156 // BinarySpliiter(BO)(Builder, X, Y, Name) uses Builder to create
157 // a binary operator like BO called Name with operands X and Y.
158 struct BinarySplitter {
159  BinarySplitter(BinaryOperator &bo) : BO(bo) {}
160 
161  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
162  const Twine &Name) const {
163  return Builder.CreateBinOp(BO.getOpcode(), Op0, Op1, Name);
164  }
165 
166  BinaryOperator &BO;
167 };
168 
169 // Information about a load or store that we're scalarizing.
170 struct VectorLayout {
171  VectorLayout() = default;
172 
173  // Return the alignment of element I.
174  Align getElemAlign(unsigned I) {
175  return commonAlignment(VecAlign, I * ElemSize);
176  }
177 
178  // The type of the vector.
179  VectorType *VecTy = nullptr;
180 
181  // The type of each element.
182  Type *ElemTy = nullptr;
183 
184  // The alignment of the vector.
185  Align VecAlign;
186 
187  // The size of each element.
188  uint64_t ElemSize = 0;
189 };
190 
191 class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
192 public:
193  ScalarizerVisitor(unsigned ParallelLoopAccessMDKind, DominatorTree *DT)
194  : ParallelLoopAccessMDKind(ParallelLoopAccessMDKind), DT(DT) {
195  }
196 
197  bool visit(Function &F);
198 
199  // InstVisitor methods. They return true if the instruction was scalarized,
200  // false if nothing changed.
201  bool visitInstruction(Instruction &I) { return false; }
202  bool visitSelectInst(SelectInst &SI);
203  bool visitICmpInst(ICmpInst &ICI);
204  bool visitFCmpInst(FCmpInst &FCI);
205  bool visitUnaryOperator(UnaryOperator &UO);
206  bool visitBinaryOperator(BinaryOperator &BO);
207  bool visitGetElementPtrInst(GetElementPtrInst &GEPI);
208  bool visitCastInst(CastInst &CI);
209  bool visitBitCastInst(BitCastInst &BCI);
210  bool visitInsertElementInst(InsertElementInst &IEI);
211  bool visitExtractElementInst(ExtractElementInst &EEI);
212  bool visitShuffleVectorInst(ShuffleVectorInst &SVI);
213  bool visitPHINode(PHINode &PHI);
214  bool visitLoadInst(LoadInst &LI);
215  bool visitStoreInst(StoreInst &SI);
216  bool visitCallInst(CallInst &ICI);
217 
218 private:
219  Scatterer scatter(Instruction *Point, Value *V);
220  void gather(Instruction *Op, const ValueVector &CV);
221  bool canTransferMetadata(unsigned Kind);
222  void transferMetadataAndIRFlags(Instruction *Op, const ValueVector &CV);
223  Optional<VectorLayout> getVectorLayout(Type *Ty, Align Alignment,
224  const DataLayout &DL);
225  bool finish();
226 
227  template<typename T> bool splitUnary(Instruction &, const T &);
228  template<typename T> bool splitBinary(Instruction &, const T &);
229 
230  bool splitCall(CallInst &CI);
231 
232  ScatterMap Scattered;
233  GatherList Gathered;
234 
235  SmallVector<WeakTrackingVH, 32> PotentiallyDeadInstrs;
236 
237  unsigned ParallelLoopAccessMDKind;
238 
239  DominatorTree *DT;
240 };
241 
242 class ScalarizerLegacyPass : public FunctionPass {
243 public:
244  static char ID;
245 
246  ScalarizerLegacyPass() : FunctionPass(ID) {
248  }
249 
250  bool runOnFunction(Function &F) override;
251 
252  void getAnalysisUsage(AnalysisUsage& AU) const override {
255  }
256 };
257 
258 } // end anonymous namespace
259 
260 char ScalarizerLegacyPass::ID = 0;
261 INITIALIZE_PASS_BEGIN(ScalarizerLegacyPass, "scalarizer",
262  "Scalarize vector operations", false, false)
264 INITIALIZE_PASS_END(ScalarizerLegacyPass, "scalarizer",
265  "Scalarize vector operations", false, false)
266 
267 Scatterer::Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v,
268  ValueVector *cachePtr)
269  : BB(bb), BBI(bbi), V(v), CachePtr(cachePtr) {
270  Type *Ty = V->getType();
271  PtrTy = dyn_cast<PointerType>(Ty);
272  if (PtrTy)
273  Ty = PtrTy->getElementType();
274  Size = cast<FixedVectorType>(Ty)->getNumElements();
275  if (!CachePtr)
276  Tmp.resize(Size, nullptr);
277  else if (CachePtr->empty())
278  CachePtr->resize(Size, nullptr);
279  else
280  assert(Size == CachePtr->size() && "Inconsistent vector sizes");
281 }
282 
283 // Return component I, creating a new Value for it if necessary.
284 Value *Scatterer::operator[](unsigned I) {
285  ValueVector &CV = (CachePtr ? *CachePtr : Tmp);
286  // Try to reuse a previous value.
287  if (CV[I])
288  return CV[I];
289  IRBuilder<> Builder(BB, BBI);
290  if (PtrTy) {
291  Type *ElTy = cast<VectorType>(PtrTy->getElementType())->getElementType();
292  if (!CV[0]) {
293  Type *NewPtrTy = PointerType::get(ElTy, PtrTy->getAddressSpace());
294  CV[0] = Builder.CreateBitCast(V, NewPtrTy, V->getName() + ".i0");
295  }
296  if (I != 0)
297  CV[I] = Builder.CreateConstGEP1_32(ElTy, CV[0], I,
298  V->getName() + ".i" + Twine(I));
299  } else {
300  // Search through a chain of InsertElementInsts looking for element I.
301  // Record other elements in the cache. The new V is still suitable
302  // for all uncached indices.
303  while (true) {
304  InsertElementInst *Insert = dyn_cast<InsertElementInst>(V);
305  if (!Insert)
306  break;
307  ConstantInt *Idx = dyn_cast<ConstantInt>(Insert->getOperand(2));
308  if (!Idx)
309  break;
310  unsigned J = Idx->getZExtValue();
311  V = Insert->getOperand(0);
312  if (I == J) {
313  CV[J] = Insert->getOperand(1);
314  return CV[J];
315  } else if (!CV[J]) {
316  // Only cache the first entry we find for each index we're not actively
317  // searching for. This prevents us from going too far up the chain and
318  // caching incorrect entries.
319  CV[J] = Insert->getOperand(1);
320  }
321  }
322  CV[I] = Builder.CreateExtractElement(V, Builder.getInt32(I),
323  V->getName() + ".i" + Twine(I));
324  }
325  return CV[I];
326 }
327 
329  if (skipFunction(F))
330  return false;
331 
332  Module &M = *F.getParent();
333  unsigned ParallelLoopAccessMDKind =
334  M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
335  DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
336  ScalarizerVisitor Impl(ParallelLoopAccessMDKind, DT);
337  return Impl.visit(F);
338 }
339 
341  return new ScalarizerLegacyPass();
342 }
343 
344 bool ScalarizerVisitor::visit(Function &F) {
345  assert(Gathered.empty() && Scattered.empty());
346 
347  // To ensure we replace gathered components correctly we need to do an ordered
348  // traversal of the basic blocks in the function.
349  ReversePostOrderTraversal<BasicBlock *> RPOT(&F.getEntryBlock());
350  for (BasicBlock *BB : RPOT) {
351  for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
352  Instruction *I = &*II;
353  bool Done = InstVisitor::visit(I);
354  ++II;
355  if (Done && I->getType()->isVoidTy())
356  I->eraseFromParent();
357  }
358  }
359  return finish();
360 }
361 
362 // Return a scattered form of V that can be accessed by Point. V must be a
363 // vector or a pointer to a vector.
364 Scatterer ScalarizerVisitor::scatter(Instruction *Point, Value *V) {
365  if (Argument *VArg = dyn_cast<Argument>(V)) {
366  // Put the scattered form of arguments in the entry block,
367  // so that it can be used everywhere.
368  Function *F = VArg->getParent();
369  BasicBlock *BB = &F->getEntryBlock();
370  return Scatterer(BB, BB->begin(), V, &Scattered[V]);
371  }
372  if (Instruction *VOp = dyn_cast<Instruction>(V)) {
373  // When scalarizing PHI nodes we might try to examine/rewrite InsertElement
374  // nodes in predecessors. If those predecessors are unreachable from entry,
375  // then the IR in those blocks could have unexpected properties resulting in
376  // infinite loops in Scatterer::operator[]. By simply treating values
377  // originating from instructions in unreachable blocks as undef we do not
378  // need to analyse them further.
379  if (!DT->isReachableFromEntry(VOp->getParent()))
380  return Scatterer(Point->getParent(), Point->getIterator(),
381  UndefValue::get(V->getType()));
382  // Put the scattered form of an instruction directly after the
383  // instruction, skipping over PHI nodes and debug intrinsics.
384  BasicBlock *BB = VOp->getParent();
385  return Scatterer(
386  BB, skipPastPhiNodesAndDbg(std::next(BasicBlock::iterator(VOp))), V,
387  &Scattered[V]);
388  }
389  // In the fallback case, just put the scattered before Point and
390  // keep the result local to Point.
391  return Scatterer(Point->getParent(), Point->getIterator(), V);
392 }
393 
394 // Replace Op with the gathered form of the components in CV. Defer the
395 // deletion of Op and creation of the gathered form to the end of the pass,
396 // so that we can avoid creating the gathered form if all uses of Op are
397 // replaced with uses of CV.
398 void ScalarizerVisitor::gather(Instruction *Op, const ValueVector &CV) {
399  transferMetadataAndIRFlags(Op, CV);
400 
401  // If we already have a scattered form of Op (created from ExtractElements
402  // of Op itself), replace them with the new form.
403  ValueVector &SV = Scattered[Op];
404  if (!SV.empty()) {
405  for (unsigned I = 0, E = SV.size(); I != E; ++I) {
406  Value *V = SV[I];
407  if (V == nullptr || SV[I] == CV[I])
408  continue;
409 
410  Instruction *Old = cast<Instruction>(V);
411  if (isa<Instruction>(CV[I]))
412  CV[I]->takeName(Old);
413  Old->replaceAllUsesWith(CV[I]);
414  PotentiallyDeadInstrs.emplace_back(Old);
415  }
416  }
417  SV = CV;
418  Gathered.push_back(GatherList::value_type(Op, &SV));
419 }
420 
421 // Return true if it is safe to transfer the given metadata tag from
422 // vector to scalar instructions.
423 bool ScalarizerVisitor::canTransferMetadata(unsigned Tag) {
424  return (Tag == LLVMContext::MD_tbaa
425  || Tag == LLVMContext::MD_fpmath
426  || Tag == LLVMContext::MD_tbaa_struct
427  || Tag == LLVMContext::MD_invariant_load
428  || Tag == LLVMContext::MD_alias_scope
429  || Tag == LLVMContext::MD_noalias
430  || Tag == ParallelLoopAccessMDKind
431  || Tag == LLVMContext::MD_access_group);
432 }
433 
434 // Transfer metadata from Op to the instructions in CV if it is known
435 // to be safe to do so.
436 void ScalarizerVisitor::transferMetadataAndIRFlags(Instruction *Op,
437  const ValueVector &CV) {
439  Op->getAllMetadataOtherThanDebugLoc(MDs);
440  for (unsigned I = 0, E = CV.size(); I != E; ++I) {
441  if (Instruction *New = dyn_cast<Instruction>(CV[I])) {
442  for (const auto &MD : MDs)
443  if (canTransferMetadata(MD.first))
444  New->setMetadata(MD.first, MD.second);
445  New->copyIRFlags(Op);
446  if (Op->getDebugLoc() && !New->getDebugLoc())
447  New->setDebugLoc(Op->getDebugLoc());
448  }
449  }
450 }
451 
452 // Try to fill in Layout from Ty, returning true on success. Alignment is
453 // the alignment of the vector, or None if the ABI default should be used.
455 ScalarizerVisitor::getVectorLayout(Type *Ty, Align Alignment,
456  const DataLayout &DL) {
457  VectorLayout Layout;
458  // Make sure we're dealing with a vector.
459  Layout.VecTy = dyn_cast<VectorType>(Ty);
460  if (!Layout.VecTy)
461  return None;
462  // Check that we're dealing with full-byte elements.
463  Layout.ElemTy = Layout.VecTy->getElementType();
464  if (!DL.typeSizeEqualsStoreSize(Layout.ElemTy))
465  return None;
466  Layout.VecAlign = Alignment;
467  Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy);
468  return Layout;
469 }
470 
471 // Scalarize one-operand instruction I, using Split(Builder, X, Name)
472 // to create an instruction like I with operand X and name Name.
473 template<typename Splitter>
474 bool ScalarizerVisitor::splitUnary(Instruction &I, const Splitter &Split) {
475  VectorType *VT = dyn_cast<VectorType>(I.getType());
476  if (!VT)
477  return false;
478 
479  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
481  Scatterer Op = scatter(&I, I.getOperand(0));
482  assert(Op.size() == NumElems && "Mismatched unary operation");
483  ValueVector Res;
484  Res.resize(NumElems);
485  for (unsigned Elem = 0; Elem < NumElems; ++Elem)
486  Res[Elem] = Split(Builder, Op[Elem], I.getName() + ".i" + Twine(Elem));
487  gather(&I, Res);
488  return true;
489 }
490 
491 // Scalarize two-operand instruction I, using Split(Builder, X, Y, Name)
492 // to create an instruction like I with operands X and Y and name Name.
493 template<typename Splitter>
494 bool ScalarizerVisitor::splitBinary(Instruction &I, const Splitter &Split) {
495  VectorType *VT = dyn_cast<VectorType>(I.getType());
496  if (!VT)
497  return false;
498 
499  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
501  Scatterer VOp0 = scatter(&I, I.getOperand(0));
502  Scatterer VOp1 = scatter(&I, I.getOperand(1));
503  assert(VOp0.size() == NumElems && "Mismatched binary operation");
504  assert(VOp1.size() == NumElems && "Mismatched binary operation");
505  ValueVector Res;
506  Res.resize(NumElems);
507  for (unsigned Elem = 0; Elem < NumElems; ++Elem) {
508  Value *Op0 = VOp0[Elem];
509  Value *Op1 = VOp1[Elem];
510  Res[Elem] = Split(Builder, Op0, Op1, I.getName() + ".i" + Twine(Elem));
511  }
512  gather(&I, Res);
513  return true;
514 }
515 
517  return isTriviallyVectorizable(ID);
518 }
519 
520 // All of the current scalarizable intrinsics only have one mangled type.
523  ArrayRef<Type*> Tys) {
524  return Intrinsic::getDeclaration(M, ID, Tys);
525 }
526 
527 /// If a call to a vector typed intrinsic function, split into a scalar call per
528 /// element if possible for the intrinsic.
529 bool ScalarizerVisitor::splitCall(CallInst &CI) {
530  VectorType *VT = dyn_cast<VectorType>(CI.getType());
531  if (!VT)
532  return false;
533 
534  Function *F = CI.getCalledFunction();
535  if (!F)
536  return false;
537 
538  Intrinsic::ID ID = F->getIntrinsicID();
540  return false;
541 
542  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
543  unsigned NumArgs = CI.arg_size();
544 
545  ValueVector ScalarOperands(NumArgs);
546  SmallVector<Scatterer, 8> Scattered(NumArgs);
547 
548  Scattered.resize(NumArgs);
549 
551  Tys.push_back(VT->getScalarType());
552 
553  // Assumes that any vector type has the same number of elements as the return
554  // vector type, which is true for all current intrinsics.
555  for (unsigned I = 0; I != NumArgs; ++I) {
556  Value *OpI = CI.getOperand(I);
557  if (OpI->getType()->isVectorTy()) {
558  Scattered[I] = scatter(&CI, OpI);
559  assert(Scattered[I].size() == NumElems && "mismatched call operands");
560  } else {
561  ScalarOperands[I] = OpI;
563  Tys.push_back(OpI->getType());
564  }
565  }
566 
567  ValueVector Res(NumElems);
568  ValueVector ScalarCallOps(NumArgs);
569 
570  Function *NewIntrin = getScalarIntrinsicDeclaration(F->getParent(), ID, Tys);
571  IRBuilder<> Builder(&CI);
572 
573  // Perform actual scalarization, taking care to preserve any scalar operands.
574  for (unsigned Elem = 0; Elem < NumElems; ++Elem) {
575  ScalarCallOps.clear();
576 
577  for (unsigned J = 0; J != NumArgs; ++J) {
579  ScalarCallOps.push_back(ScalarOperands[J]);
580  else
581  ScalarCallOps.push_back(Scattered[J][Elem]);
582  }
583 
584  Res[Elem] = Builder.CreateCall(NewIntrin, ScalarCallOps,
585  CI.getName() + ".i" + Twine(Elem));
586  }
587 
588  gather(&CI, Res);
589  return true;
590 }
591 
592 bool ScalarizerVisitor::visitSelectInst(SelectInst &SI) {
593  VectorType *VT = dyn_cast<VectorType>(SI.getType());
594  if (!VT)
595  return false;
596 
597  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
599  Scatterer VOp1 = scatter(&SI, SI.getOperand(1));
600  Scatterer VOp2 = scatter(&SI, SI.getOperand(2));
601  assert(VOp1.size() == NumElems && "Mismatched select");
602  assert(VOp2.size() == NumElems && "Mismatched select");
603  ValueVector Res;
604  Res.resize(NumElems);
605 
606  if (SI.getOperand(0)->getType()->isVectorTy()) {
607  Scatterer VOp0 = scatter(&SI, SI.getOperand(0));
608  assert(VOp0.size() == NumElems && "Mismatched select");
609  for (unsigned I = 0; I < NumElems; ++I) {
610  Value *Op0 = VOp0[I];
611  Value *Op1 = VOp1[I];
612  Value *Op2 = VOp2[I];
613  Res[I] = Builder.CreateSelect(Op0, Op1, Op2,
614  SI.getName() + ".i" + Twine(I));
615  }
616  } else {
617  Value *Op0 = SI.getOperand(0);
618  for (unsigned I = 0; I < NumElems; ++I) {
619  Value *Op1 = VOp1[I];
620  Value *Op2 = VOp2[I];
621  Res[I] = Builder.CreateSelect(Op0, Op1, Op2,
622  SI.getName() + ".i" + Twine(I));
623  }
624  }
625  gather(&SI, Res);
626  return true;
627 }
628 
629 bool ScalarizerVisitor::visitICmpInst(ICmpInst &ICI) {
630  return splitBinary(ICI, ICmpSplitter(ICI));
631 }
632 
633 bool ScalarizerVisitor::visitFCmpInst(FCmpInst &FCI) {
634  return splitBinary(FCI, FCmpSplitter(FCI));
635 }
636 
637 bool ScalarizerVisitor::visitUnaryOperator(UnaryOperator &UO) {
638  return splitUnary(UO, UnarySplitter(UO));
639 }
640 
641 bool ScalarizerVisitor::visitBinaryOperator(BinaryOperator &BO) {
642  return splitBinary(BO, BinarySplitter(BO));
643 }
644 
645 bool ScalarizerVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
646  VectorType *VT = dyn_cast<VectorType>(GEPI.getType());
647  if (!VT)
648  return false;
649 
650  IRBuilder<> Builder(&GEPI);
651  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
652  unsigned NumIndices = GEPI.getNumIndices();
653 
654  // The base pointer might be scalar even if it's a vector GEP. In those cases,
655  // splat the pointer into a vector value, and scatter that vector.
656  Value *Op0 = GEPI.getOperand(0);
657  if (!Op0->getType()->isVectorTy())
658  Op0 = Builder.CreateVectorSplat(NumElems, Op0);
659  Scatterer Base = scatter(&GEPI, Op0);
660 
662  Ops.resize(NumIndices);
663  for (unsigned I = 0; I < NumIndices; ++I) {
664  Value *Op = GEPI.getOperand(I + 1);
665 
666  // The indices might be scalars even if it's a vector GEP. In those cases,
667  // splat the scalar into a vector value, and scatter that vector.
668  if (!Op->getType()->isVectorTy())
669  Op = Builder.CreateVectorSplat(NumElems, Op);
670 
671  Ops[I] = scatter(&GEPI, Op);
672  }
673 
674  ValueVector Res;
675  Res.resize(NumElems);
676  for (unsigned I = 0; I < NumElems; ++I) {
677  SmallVector<Value *, 8> Indices;
678  Indices.resize(NumIndices);
679  for (unsigned J = 0; J < NumIndices; ++J)
680  Indices[J] = Ops[J][I];
681  Res[I] = Builder.CreateGEP(GEPI.getSourceElementType(), Base[I], Indices,
682  GEPI.getName() + ".i" + Twine(I));
683  if (GEPI.isInBounds())
684  if (GetElementPtrInst *NewGEPI = dyn_cast<GetElementPtrInst>(Res[I]))
685  NewGEPI->setIsInBounds();
686  }
687  gather(&GEPI, Res);
688  return true;
689 }
690 
691 bool ScalarizerVisitor::visitCastInst(CastInst &CI) {
692  VectorType *VT = dyn_cast<VectorType>(CI.getDestTy());
693  if (!VT)
694  return false;
695 
696  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
697  IRBuilder<> Builder(&CI);
698  Scatterer Op0 = scatter(&CI, CI.getOperand(0));
699  assert(Op0.size() == NumElems && "Mismatched cast");
700  ValueVector Res;
701  Res.resize(NumElems);
702  for (unsigned I = 0; I < NumElems; ++I)
703  Res[I] = Builder.CreateCast(CI.getOpcode(), Op0[I], VT->getElementType(),
704  CI.getName() + ".i" + Twine(I));
705  gather(&CI, Res);
706  return true;
707 }
708 
709 bool ScalarizerVisitor::visitBitCastInst(BitCastInst &BCI) {
710  VectorType *DstVT = dyn_cast<VectorType>(BCI.getDestTy());
711  VectorType *SrcVT = dyn_cast<VectorType>(BCI.getSrcTy());
712  if (!DstVT || !SrcVT)
713  return false;
714 
715  unsigned DstNumElems = cast<FixedVectorType>(DstVT)->getNumElements();
716  unsigned SrcNumElems = cast<FixedVectorType>(SrcVT)->getNumElements();
717  IRBuilder<> Builder(&BCI);
718  Scatterer Op0 = scatter(&BCI, BCI.getOperand(0));
719  ValueVector Res;
720  Res.resize(DstNumElems);
721 
722  if (DstNumElems == SrcNumElems) {
723  for (unsigned I = 0; I < DstNumElems; ++I)
724  Res[I] = Builder.CreateBitCast(Op0[I], DstVT->getElementType(),
725  BCI.getName() + ".i" + Twine(I));
726  } else if (DstNumElems > SrcNumElems) {
727  // <M x t1> -> <N*M x t2>. Convert each t1 to <N x t2> and copy the
728  // individual elements to the destination.
729  unsigned FanOut = DstNumElems / SrcNumElems;
730  auto *MidTy = FixedVectorType::get(DstVT->getElementType(), FanOut);
731  unsigned ResI = 0;
732  for (unsigned Op0I = 0; Op0I < SrcNumElems; ++Op0I) {
733  Value *V = Op0[Op0I];
734  Instruction *VI;
735  // Look through any existing bitcasts before converting to <N x t2>.
736  // In the best case, the resulting conversion might be a no-op.
737  while ((VI = dyn_cast<Instruction>(V)) &&
738  VI->getOpcode() == Instruction::BitCast)
739  V = VI->getOperand(0);
740  V = Builder.CreateBitCast(V, MidTy, V->getName() + ".cast");
741  Scatterer Mid = scatter(&BCI, V);
742  for (unsigned MidI = 0; MidI < FanOut; ++MidI)
743  Res[ResI++] = Mid[MidI];
744  }
745  } else {
746  // <N*M x t1> -> <M x t2>. Convert each group of <N x t1> into a t2.
747  unsigned FanIn = SrcNumElems / DstNumElems;
748  auto *MidTy = FixedVectorType::get(SrcVT->getElementType(), FanIn);
749  unsigned Op0I = 0;
750  for (unsigned ResI = 0; ResI < DstNumElems; ++ResI) {
751  Value *V = PoisonValue::get(MidTy);
752  for (unsigned MidI = 0; MidI < FanIn; ++MidI)
753  V = Builder.CreateInsertElement(V, Op0[Op0I++], Builder.getInt32(MidI),
754  BCI.getName() + ".i" + Twine(ResI)
755  + ".upto" + Twine(MidI));
756  Res[ResI] = Builder.CreateBitCast(V, DstVT->getElementType(),
757  BCI.getName() + ".i" + Twine(ResI));
758  }
759  }
760  gather(&BCI, Res);
761  return true;
762 }
763 
764 bool ScalarizerVisitor::visitInsertElementInst(InsertElementInst &IEI) {
765  VectorType *VT = dyn_cast<VectorType>(IEI.getType());
766  if (!VT)
767  return false;
768 
769  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
770  IRBuilder<> Builder(&IEI);
771  Scatterer Op0 = scatter(&IEI, IEI.getOperand(0));
772  Value *NewElt = IEI.getOperand(1);
773  Value *InsIdx = IEI.getOperand(2);
774 
775  ValueVector Res;
776  Res.resize(NumElems);
777 
778  if (auto *CI = dyn_cast<ConstantInt>(InsIdx)) {
779  for (unsigned I = 0; I < NumElems; ++I)
780  Res[I] = CI->getValue().getZExtValue() == I ? NewElt : Op0[I];
781  } else {
783  return false;
784 
785  for (unsigned I = 0; I < NumElems; ++I) {
786  Value *ShouldReplace =
787  Builder.CreateICmpEQ(InsIdx, ConstantInt::get(InsIdx->getType(), I),
788  InsIdx->getName() + ".is." + Twine(I));
789  Value *OldElt = Op0[I];
790  Res[I] = Builder.CreateSelect(ShouldReplace, NewElt, OldElt,
791  IEI.getName() + ".i" + Twine(I));
792  }
793  }
794 
795  gather(&IEI, Res);
796  return true;
797 }
798 
799 bool ScalarizerVisitor::visitExtractElementInst(ExtractElementInst &EEI) {
800  VectorType *VT = dyn_cast<VectorType>(EEI.getOperand(0)->getType());
801  if (!VT)
802  return false;
803 
804  unsigned NumSrcElems = cast<FixedVectorType>(VT)->getNumElements();
805  IRBuilder<> Builder(&EEI);
806  Scatterer Op0 = scatter(&EEI, EEI.getOperand(0));
807  Value *ExtIdx = EEI.getOperand(1);
808 
809  if (auto *CI = dyn_cast<ConstantInt>(ExtIdx)) {
810  Value *Res = Op0[CI->getValue().getZExtValue()];
811  gather(&EEI, {Res});
812  return true;
813  }
814 
816  return false;
817 
818  Value *Res = UndefValue::get(VT->getElementType());
819  for (unsigned I = 0; I < NumSrcElems; ++I) {
820  Value *ShouldExtract =
821  Builder.CreateICmpEQ(ExtIdx, ConstantInt::get(ExtIdx->getType(), I),
822  ExtIdx->getName() + ".is." + Twine(I));
823  Value *Elt = Op0[I];
824  Res = Builder.CreateSelect(ShouldExtract, Elt, Res,
825  EEI.getName() + ".upto" + Twine(I));
826  }
827  gather(&EEI, {Res});
828  return true;
829 }
830 
831 bool ScalarizerVisitor::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
832  VectorType *VT = dyn_cast<VectorType>(SVI.getType());
833  if (!VT)
834  return false;
835 
836  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
837  Scatterer Op0 = scatter(&SVI, SVI.getOperand(0));
838  Scatterer Op1 = scatter(&SVI, SVI.getOperand(1));
839  ValueVector Res;
840  Res.resize(NumElems);
841 
842  for (unsigned I = 0; I < NumElems; ++I) {
843  int Selector = SVI.getMaskValue(I);
844  if (Selector < 0)
845  Res[I] = UndefValue::get(VT->getElementType());
846  else if (unsigned(Selector) < Op0.size())
847  Res[I] = Op0[Selector];
848  else
849  Res[I] = Op1[Selector - Op0.size()];
850  }
851  gather(&SVI, Res);
852  return true;
853 }
854 
855 bool ScalarizerVisitor::visitPHINode(PHINode &PHI) {
856  VectorType *VT = dyn_cast<VectorType>(PHI.getType());
857  if (!VT)
858  return false;
859 
860  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
861  IRBuilder<> Builder(&PHI);
862  ValueVector Res;
863  Res.resize(NumElems);
864 
865  unsigned NumOps = PHI.getNumOperands();
866  for (unsigned I = 0; I < NumElems; ++I)
867  Res[I] = Builder.CreatePHI(VT->getElementType(), NumOps,
868  PHI.getName() + ".i" + Twine(I));
869 
870  for (unsigned I = 0; I < NumOps; ++I) {
871  Scatterer Op = scatter(&PHI, PHI.getIncomingValue(I));
872  BasicBlock *IncomingBlock = PHI.getIncomingBlock(I);
873  for (unsigned J = 0; J < NumElems; ++J)
874  cast<PHINode>(Res[J])->addIncoming(Op[J], IncomingBlock);
875  }
876  gather(&PHI, Res);
877  return true;
878 }
879 
880 bool ScalarizerVisitor::visitLoadInst(LoadInst &LI) {
881  if (!ScalarizeLoadStore)
882  return false;
883  if (!LI.isSimple())
884  return false;
885 
886  Optional<VectorLayout> Layout = getVectorLayout(
887  LI.getType(), LI.getAlign(), LI.getModule()->getDataLayout());
888  if (!Layout)
889  return false;
890 
891  unsigned NumElems = cast<FixedVectorType>(Layout->VecTy)->getNumElements();
892  IRBuilder<> Builder(&LI);
893  Scatterer Ptr = scatter(&LI, LI.getPointerOperand());
894  ValueVector Res;
895  Res.resize(NumElems);
896 
897  for (unsigned I = 0; I < NumElems; ++I)
898  Res[I] = Builder.CreateAlignedLoad(Layout->VecTy->getElementType(), Ptr[I],
899  Align(Layout->getElemAlign(I)),
900  LI.getName() + ".i" + Twine(I));
901  gather(&LI, Res);
902  return true;
903 }
904 
905 bool ScalarizerVisitor::visitStoreInst(StoreInst &SI) {
906  if (!ScalarizeLoadStore)
907  return false;
908  if (!SI.isSimple())
909  return false;
910 
911  Value *FullValue = SI.getValueOperand();
912  Optional<VectorLayout> Layout = getVectorLayout(
913  FullValue->getType(), SI.getAlign(), SI.getModule()->getDataLayout());
914  if (!Layout)
915  return false;
916 
917  unsigned NumElems = cast<FixedVectorType>(Layout->VecTy)->getNumElements();
919  Scatterer VPtr = scatter(&SI, SI.getPointerOperand());
920  Scatterer VVal = scatter(&SI, FullValue);
921 
922  ValueVector Stores;
923  Stores.resize(NumElems);
924  for (unsigned I = 0; I < NumElems; ++I) {
925  Value *Val = VVal[I];
926  Value *Ptr = VPtr[I];
927  Stores[I] = Builder.CreateAlignedStore(Val, Ptr, Layout->getElemAlign(I));
928  }
929  transferMetadataAndIRFlags(&SI, Stores);
930  return true;
931 }
932 
933 bool ScalarizerVisitor::visitCallInst(CallInst &CI) {
934  return splitCall(CI);
935 }
936 
937 // Delete the instructions that we scalarized. If a full vector result
938 // is still needed, recreate it using InsertElements.
939 bool ScalarizerVisitor::finish() {
940  // The presence of data in Gathered or Scattered indicates changes
941  // made to the Function.
942  if (Gathered.empty() && Scattered.empty())
943  return false;
944  for (const auto &GMI : Gathered) {
945  Instruction *Op = GMI.first;
946  ValueVector &CV = *GMI.second;
947  if (!Op->use_empty()) {
948  // The value is still needed, so recreate it using a series of
949  // InsertElements.
950  Value *Res = PoisonValue::get(Op->getType());
951  if (auto *Ty = dyn_cast<VectorType>(Op->getType())) {
952  BasicBlock *BB = Op->getParent();
953  unsigned Count = cast<FixedVectorType>(Ty)->getNumElements();
955  if (isa<PHINode>(Op))
956  Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
957  for (unsigned I = 0; I < Count; ++I)
958  Res = Builder.CreateInsertElement(Res, CV[I], Builder.getInt32(I),
959  Op->getName() + ".upto" + Twine(I));
960  Res->takeName(Op);
961  } else {
962  assert(CV.size() == 1 && Op->getType() == CV[0]->getType());
963  Res = CV[0];
964  if (Op == Res)
965  continue;
966  }
967  Op->replaceAllUsesWith(Res);
968  }
969  PotentiallyDeadInstrs.emplace_back(Op);
970  }
971  Gathered.clear();
972  Scattered.clear();
973 
975 
976  return true;
977 }
978 
980  Module &M = *F.getParent();
981  unsigned ParallelLoopAccessMDKind =
982  M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
984  ScalarizerVisitor Impl(ParallelLoopAccessMDKind, DT);
985  bool Changed = Impl.visit(F);
988  return Changed ? PA : PreservedAnalyses::all();
989 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
MathExtras.h
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AllocatorList.h:23
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:66
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1399
Insert
Vector Rotate Left Mask Mask Insert
Definition: README_P9.txt:112
llvm::BasicBlock::iterator
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:90
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:783
Scalar.h
T
llvm::ExtractElementInst
This instruction extracts a single (scalar) element from a VectorType value.
Definition: Instructions.h:1867
llvm::Function
Definition: Function.h:62
Pass.h
llvm::PointerType::get
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Definition: Type.cpp:729
llvm::BitCastInst
This class represents a no-op cast from one type to another.
Definition: Instructions.h:5209
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:308
llvm::SmallVector< Value *, 8 >
llvm::IRBuilder<>
Local.h
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:151
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Module.h
llvm::Optional
Definition: APInt.h:33
llvm::GetElementPtrInst::getNumIndices
unsigned getNumIndices() const
Definition: Instructions.h:1096
llvm::VectorType::getElementType
Type * getElementType() const
Definition: DerivedTypes.h:422
llvm::Intrinsic::not_intrinsic
@ not_intrinsic
Definition: Intrinsics.h:45
llvm::LoadInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:267
llvm::UnaryOperator
Definition: InstrTypes.h:103
getScalarIntrinsicDeclaration
static Function * getScalarIntrinsicDeclaration(Module *M, Intrinsic::ID ID, ArrayRef< Type * > Tys)
Definition: Scalarizer.cpp:521
llvm::LoadInst::getAlign
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:223
llvm::CastInst::getDestTy
Type * getDestTy() const
Return the destination type, as a convenience.
Definition: InstrTypes.h:685
llvm::GetElementPtrInst::getSourceElementType
Type * getSourceElementType() const
Definition: Instructions.h:1003
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
Scalarizer.h
llvm::GetElementPtrInst::isInBounds
bool isInBounds() const
Determine whether the GEP has the inbounds flag.
Definition: Instructions.cpp:1814
Instruction.h
CommandLine.h
operations
Scalarize vector operations
Definition: Scalarizer.cpp:265
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
bb
< i1 > br i1 label label bb bb
Definition: README.txt:978
llvm::AArch64Layout::VectorLayout
VectorLayout
Definition: AArch64BaseInfo.h:567
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
Constants.h
llvm::PHINode::getIncomingValue
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
Definition: Instructions.h:2744
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::isTriviallyVectorizable
bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
Definition: VectorUtils.cpp:44
Intrinsics.h
Twine.h
InstrTypes.h
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1398
llvm::FCmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1359
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:226
llvm::InsertElementInst
This instruction inserts a single (scalar) element into a VectorType value.
Definition: Instructions.h:1931
false
Definition: StackSlotColoring.cpp:142
llvm::ShuffleVectorInst::getMaskValue
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
Definition: Instructions.h:2058
llvm::skipDebugIntrinsics
BasicBlock::iterator skipDebugIntrinsics(BasicBlock::iterator It)
Advance It while it points to a debug instruction and return the result.
Definition: BasicBlock.cpp:497
llvm::Instruction
Definition: Instruction.h:45
llvm::DominatorTreeWrapperPass
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:287
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:606
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1796
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:925
llvm::ScalarizerPass::run
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Definition: Scalarizer.cpp:979
llvm::FixedVectorType::get
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:686
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::CastInst::getSrcTy
Type * getSrcTy() const
Return the source type, as a convenience.
Definition: InstrTypes.h:683
llvm::None
const NoneType None
Definition: None.h:23
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
llvm::ShuffleVectorInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:2049
Type.h
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
llvm::ARM_PROC::IE
@ IE
Definition: ARMBaseInfo.h:27
llvm::VectorType
Base class of all SIMD vector types.
Definition: DerivedTypes.h:389
VectorUtils.h
BasicBlock.h
llvm::cl::opt< bool >
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:304
VI
@ VI
Definition: SIInstrInfo.cpp:7721
ScalarizeLoadStore
static cl::opt< bool > ScalarizeLoadStore("scalarize-load-store", cl::init(false), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize loads and store"))
llvm::hasVectorInstrinsicOverloadedScalarOpd
bool hasVectorInstrinsicOverloadedScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand that has an overloaded type.
Definition: VectorUtils.cpp:117
llvm::ICmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1185
uint64_t
llvm::PreservedAnalyses::preserve
void preserve()
Mark an analysis as preserved.
Definition: PassManager.h:176
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::GetElementPtrInst
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:928
llvm::hasVectorInstrinsicScalarOpd
bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
Definition: VectorUtils.cpp:99
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:441
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:632
llvm::InstVisitor::visit
void visit(Iterator Start, Iterator End)
Definition: InstVisitor.h:88
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::createScalarizerPass
FunctionPass * createScalarizerPass()
Create a legacy pass manager instance of the Scalarizer pass.
Definition: Scalarizer.cpp:340
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::DominatorTree::isReachableFromEntry
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
Definition: Dominators.cpp:328
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1732
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:650
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1581
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::BinaryOperator
Definition: InstrTypes.h:190
DataLayout.h
InstVisitor.h
llvm::LoadInst::isSimple
bool isSimple() const
Definition: Instructions.h:259
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::AnalysisUsage::addPreserved
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Definition: PassAnalysisSupport.h:98
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
llvm::ms_demangle::IntrinsicFunctionKind::New
@ New
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::initializeScalarizerLegacyPassPass
void initializeScalarizerLegacyPassPass(PassRegistry &)
llvm::InstVisitor
Base class for instruction visitors.
Definition: InstVisitor.h:79
llvm::CastInst
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:431
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:175
Argument.h
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:142
runOnFunction
static bool runOnFunction(Function &F, bool PostInlining)
Definition: EntryExitInstrumenter.cpp:69
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::commonAlignment
Align commonAlignment(Align A, Align B)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:211
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:325
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
llvm::CallBase::arg_size
unsigned arg_size() const
Definition: InstrTypes.h:1341
Casting.h
isTriviallyScalariable
static bool isTriviallyScalariable(Intrinsic::ID ID)
Definition: Scalarizer.cpp:516
Function.h
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(ScalarizerLegacyPass, "scalarizer", "Scalarize vector operations", false, false) INITIALIZE_PASS_END(ScalarizerLegacyPass
llvm::ReversePostOrderTraversal
Definition: PostOrderIterator.h:290
scalarizer
scalarizer
Definition: Scalarizer.cpp:264
llvm::DominatorTreeAnalysis
Analysis pass which computes a DominatorTree.
Definition: Dominators.h:252
llvm::InsertElementInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:1964
llvm::ShuffleVectorInst
This instruction constructs a fixed permutation of two input vectors.
Definition: Instructions.h:2003
Instructions.h
PostOrderIterator.h
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
SmallVector.h
Dominators.h
llvm::CastInst::getOpcode
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition: InstrTypes.h:678
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::PHINode::getIncomingBlock
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Definition: Instructions.h:2764
llvm::PHINode
Definition: Instructions.h:2648
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:401
DerivedTypes.h
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:44
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1469
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
LLVMContext.h
llvm::Value::takeName
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:382
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::cl::desc
Definition: CommandLine.h:412
Value.h
InitializePasses.h
llvm::RecursivelyDeleteTriviallyDeadInstructionsPermissive
bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
Definition: Local.cpp:538
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
ScalarizeVariableInsertExtract
static cl::opt< bool > ScalarizeVariableInsertExtract("scalarize-variable-insert-extract", cl::init(true), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize " "insertelement/extractelement with variable index"))
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38
llvm::PoisonValue::get
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1815