LLVM  3.7.0
LoopIdiomRecognize.cpp
Go to the documentation of this file.
1 //===-- LoopIdiomRecognize.cpp - Loop idiom recognition -------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This pass implements an idiom recognizer that transforms simple loops into a
11 // non-loop form. In cases that this kicks in, it can be a significant
12 // performance win.
13 //
14 //===----------------------------------------------------------------------===//
15 //
16 // TODO List:
17 //
18 // Future loop memory idioms to recognize:
19 // memcmp, memmove, strlen, etc.
20 // Future floating point idioms to recognize in -ffast-math mode:
21 // fpowi
22 // Future integer operation idioms to recognize:
23 // ctpop, ctlz, cttz
24 //
25 // Beware that isel's default lowering for ctpop is highly inefficient for
26 // i64 and larger types when i64 is legal and the value has few bits set. It
27 // would be good to enhance isel to emit a loop for ctpop in this case.
28 //
29 // We should enhance the memset/memcpy recognition to handle multiple stores in
30 // the loop. This would handle things like:
31 // void foo(_Complex float *P)
32 // for (i) { __real__(*P) = 0; __imag__(*P) = 0; }
33 //
34 // We should enhance this to handle negative strides through memory.
35 // Alternatively (and perhaps better) we could rely on an earlier pass to force
36 // forward iteration through memory, which is generally better for cache
37 // behavior. Negative strides *do* happen for memset/memcpy loops.
38 //
39 // This could recognize common matrix multiplies and dot product idioms and
40 // replace them with calls to BLAS (if linked in??).
41 //
42 //===----------------------------------------------------------------------===//
43 
44 #include "llvm/Transforms/Scalar.h"
45 #include "llvm/ADT/Statistic.h"
47 #include "llvm/Analysis/LoopPass.h"
53 #include "llvm/IR/DataLayout.h"
54 #include "llvm/IR/Dominators.h"
55 #include "llvm/IR/IRBuilder.h"
56 #include "llvm/IR/IntrinsicInst.h"
57 #include "llvm/IR/Module.h"
58 #include "llvm/Support/Debug.h"
61 using namespace llvm;
62 
63 #define DEBUG_TYPE "loop-idiom"
64 
65 STATISTIC(NumMemSet, "Number of memset's formed from loop stores");
66 STATISTIC(NumMemCpy, "Number of memcpy's formed from loop load+stores");
67 
68 namespace {
69 
70  class LoopIdiomRecognize;
71 
72  /// This class defines some utility functions for loop idiom recognization.
73  class LIRUtil {
74  public:
75  /// Return true iff the block contains nothing but an uncondition branch
76  /// (aka goto instruction).
77  static bool isAlmostEmpty(BasicBlock *);
78 
79  static BranchInst *getBranch(BasicBlock *BB) {
80  return dyn_cast<BranchInst>(BB->getTerminator());
81  }
82 
83  /// Derive the precondition block (i.e the block that guards the loop
84  /// preheader) from the given preheader.
85  static BasicBlock *getPrecondBb(BasicBlock *PreHead);
86  };
87 
88  /// This class is to recoginize idioms of population-count conducted in
89  /// a noncountable loop. Currently it only recognizes this pattern:
90  /// \code
91  /// while(x) {cnt++; ...; x &= x - 1; ...}
92  /// \endcode
93  class NclPopcountRecognize {
94  LoopIdiomRecognize &LIR;
95  Loop *CurLoop;
96  BasicBlock *PreCondBB;
97 
98  typedef IRBuilder<> IRBuilderTy;
99 
100  public:
101  explicit NclPopcountRecognize(LoopIdiomRecognize &TheLIR);
102  bool recognize();
103 
104  private:
105  /// Take a glimpse of the loop to see if we need to go ahead recoginizing
106  /// the idiom.
107  bool preliminaryScreen();
108 
109  /// Check if the given conditional branch is based on the comparison
110  /// between a variable and zero, and if the variable is non-zero, the
111  /// control yields to the loop entry. If the branch matches the behavior,
112  /// the variable involved in the comparion is returned. This function will
113  /// be called to see if the precondition and postcondition of the loop
114  /// are in desirable form.
115  Value *matchCondition(BranchInst *Br, BasicBlock *NonZeroTarget) const;
116 
117  /// Return true iff the idiom is detected in the loop. and 1) \p CntInst
118  /// is set to the instruction counting the population bit. 2) \p CntPhi
119  /// is set to the corresponding phi node. 3) \p Var is set to the value
120  /// whose population bits are being counted.
121  bool detectIdiom
122  (Instruction *&CntInst, PHINode *&CntPhi, Value *&Var) const;
123 
124  /// Insert ctpop intrinsic function and some obviously dead instructions.
125  void transform(Instruction *CntInst, PHINode *CntPhi, Value *Var);
126 
127  /// Create llvm.ctpop.* intrinsic function.
128  CallInst *createPopcntIntrinsic(IRBuilderTy &IRB, Value *Val, DebugLoc DL);
129  };
130 
131  class LoopIdiomRecognize : public LoopPass {
132  Loop *CurLoop;
133  DominatorTree *DT;
134  ScalarEvolution *SE;
135  TargetLibraryInfo *TLI;
136  const TargetTransformInfo *TTI;
137  public:
138  static char ID;
139  explicit LoopIdiomRecognize() : LoopPass(ID) {
141  DT = nullptr;
142  SE = nullptr;
143  TLI = nullptr;
144  TTI = nullptr;
145  }
146 
147  bool runOnLoop(Loop *L, LPPassManager &LPM) override;
148  bool runOnLoopBlock(BasicBlock *BB, const SCEV *BECount,
149  SmallVectorImpl<BasicBlock*> &ExitBlocks);
150 
151  bool processLoopStore(StoreInst *SI, const SCEV *BECount);
152  bool processLoopMemSet(MemSetInst *MSI, const SCEV *BECount);
153 
154  bool processLoopStridedStore(Value *DestPtr, unsigned StoreSize,
155  unsigned StoreAlignment,
156  Value *SplatValue, Instruction *TheStore,
157  const SCEVAddRecExpr *Ev,
158  const SCEV *BECount);
159  bool processLoopStoreOfLoopLoad(StoreInst *SI, unsigned StoreSize,
160  const SCEVAddRecExpr *StoreEv,
161  const SCEVAddRecExpr *LoadEv,
162  const SCEV *BECount);
163 
164  /// This transformation requires natural loop information & requires that
165  /// loop preheaders be inserted into the CFG.
166  ///
167  void getAnalysisUsage(AnalysisUsage &AU) const override {
182  }
183 
184  DominatorTree *getDominatorTree() {
185  return DT ? DT
186  : (DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree());
187  }
188 
189  ScalarEvolution *getScalarEvolution() {
190  return SE ? SE : (SE = &getAnalysis<ScalarEvolution>());
191  }
192 
193  TargetLibraryInfo *getTargetLibraryInfo() {
194  if (!TLI)
195  TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
196 
197  return TLI;
198  }
199 
200  const TargetTransformInfo *getTargetTransformInfo() {
201  return TTI ? TTI
202  : (TTI = &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
203  *CurLoop->getHeader()->getParent()));
204  }
205 
206  Loop *getLoop() const { return CurLoop; }
207 
208  private:
209  bool runOnNoncountableLoop();
210  bool runOnCountableLoop();
211  };
212 }
213 
214 char LoopIdiomRecognize::ID = 0;
215 INITIALIZE_PASS_BEGIN(LoopIdiomRecognize, "loop-idiom", "Recognize loop idioms",
216  false, false)
219 INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
225 INITIALIZE_PASS_END(LoopIdiomRecognize, "loop-idiom", "Recognize loop idioms",
226  false, false)
227 
228 Pass *llvm::createLoopIdiomPass() { return new LoopIdiomRecognize(); }
229 
230 /// deleteDeadInstruction - Delete this instruction. Before we do, go through
231 /// and zero out all the operands of this instruction. If any of them become
232 /// dead, delete them and the computation tree that feeds them.
233 ///
235  const TargetLibraryInfo *TLI) {
238  I->eraseFromParent();
239  for (Value *Op : Operands)
241 }
242 
243 //===----------------------------------------------------------------------===//
244 //
245 // Implementation of LIRUtil
246 //
247 //===----------------------------------------------------------------------===//
248 
249 // This function will return true iff the given block contains nothing but goto.
250 // A typical usage of this function is to check if the preheader function is
251 // "almost" empty such that generated intrinsic functions can be moved across
252 // the preheader and be placed at the end of the precondition block without
253 // the concern of breaking data dependence.
254 bool LIRUtil::isAlmostEmpty(BasicBlock *BB) {
255  if (BranchInst *Br = getBranch(BB)) {
256  return Br->isUnconditional() && Br == BB->begin();
257  }
258  return false;
259 }
260 
261 BasicBlock *LIRUtil::getPrecondBb(BasicBlock *PreHead) {
262  if (BasicBlock *BB = PreHead->getSinglePredecessor()) {
263  BranchInst *Br = getBranch(BB);
264  return Br && Br->isConditional() ? BB : nullptr;
265  }
266  return nullptr;
267 }
268 
269 //===----------------------------------------------------------------------===//
270 //
271 // Implementation of NclPopcountRecognize
272 //
273 //===----------------------------------------------------------------------===//
274 
275 NclPopcountRecognize::NclPopcountRecognize(LoopIdiomRecognize &TheLIR):
276  LIR(TheLIR), CurLoop(TheLIR.getLoop()), PreCondBB(nullptr) {
277 }
278 
279 bool NclPopcountRecognize::preliminaryScreen() {
280  const TargetTransformInfo *TTI = LIR.getTargetTransformInfo();
281  if (TTI->getPopcntSupport(32) != TargetTransformInfo::PSK_FastHardware)
282  return false;
283 
284  // Counting population are usually conducted by few arithmetic instructions.
285  // Such instructions can be easilly "absorbed" by vacant slots in a
286  // non-compact loop. Therefore, recognizing popcount idiom only makes sense
287  // in a compact loop.
288 
289  // Give up if the loop has multiple blocks or multiple backedges.
290  if (CurLoop->getNumBackEdges() != 1 || CurLoop->getNumBlocks() != 1)
291  return false;
292 
293  BasicBlock *LoopBody = *(CurLoop->block_begin());
294  if (LoopBody->size() >= 20) {
295  // The loop is too big, bail out.
296  return false;
297  }
298 
299  // It should have a preheader containing nothing but a goto instruction.
300  BasicBlock *PreHead = CurLoop->getLoopPreheader();
301  if (!PreHead || !LIRUtil::isAlmostEmpty(PreHead))
302  return false;
303 
304  // It should have a precondition block where the generated popcount instrinsic
305  // function will be inserted.
306  PreCondBB = LIRUtil::getPrecondBb(PreHead);
307  if (!PreCondBB)
308  return false;
309 
310  return true;
311 }
312 
313 Value *NclPopcountRecognize::matchCondition(BranchInst *Br,
314  BasicBlock *LoopEntry) const {
315  if (!Br || !Br->isConditional())
316  return nullptr;
317 
318  ICmpInst *Cond = dyn_cast<ICmpInst>(Br->getCondition());
319  if (!Cond)
320  return nullptr;
321 
322  ConstantInt *CmpZero = dyn_cast<ConstantInt>(Cond->getOperand(1));
323  if (!CmpZero || !CmpZero->isZero())
324  return nullptr;
325 
326  ICmpInst::Predicate Pred = Cond->getPredicate();
327  if ((Pred == ICmpInst::ICMP_NE && Br->getSuccessor(0) == LoopEntry) ||
328  (Pred == ICmpInst::ICMP_EQ && Br->getSuccessor(1) == LoopEntry))
329  return Cond->getOperand(0);
330 
331  return nullptr;
332 }
333 
334 bool NclPopcountRecognize::detectIdiom(Instruction *&CntInst,
335  PHINode *&CntPhi,
336  Value *&Var) const {
337  // Following code tries to detect this idiom:
338  //
339  // if (x0 != 0)
340  // goto loop-exit // the precondition of the loop
341  // cnt0 = init-val;
342  // do {
343  // x1 = phi (x0, x2);
344  // cnt1 = phi(cnt0, cnt2);
345  //
346  // cnt2 = cnt1 + 1;
347  // ...
348  // x2 = x1 & (x1 - 1);
349  // ...
350  // } while(x != 0);
351  //
352  // loop-exit:
353  //
354 
355  // step 1: Check to see if the look-back branch match this pattern:
356  // "if (a!=0) goto loop-entry".
357  BasicBlock *LoopEntry;
358  Instruction *DefX2, *CountInst;
359  Value *VarX1, *VarX0;
360  PHINode *PhiX, *CountPhi;
361 
362  DefX2 = CountInst = nullptr;
363  VarX1 = VarX0 = nullptr;
364  PhiX = CountPhi = nullptr;
365  LoopEntry = *(CurLoop->block_begin());
366 
367  // step 1: Check if the loop-back branch is in desirable form.
368  {
369  if (Value *T = matchCondition (LIRUtil::getBranch(LoopEntry), LoopEntry))
370  DefX2 = dyn_cast<Instruction>(T);
371  else
372  return false;
373  }
374 
375  // step 2: detect instructions corresponding to "x2 = x1 & (x1 - 1)"
376  {
377  if (!DefX2 || DefX2->getOpcode() != Instruction::And)
378  return false;
379 
380  BinaryOperator *SubOneOp;
381 
382  if ((SubOneOp = dyn_cast<BinaryOperator>(DefX2->getOperand(0))))
383  VarX1 = DefX2->getOperand(1);
384  else {
385  VarX1 = DefX2->getOperand(0);
386  SubOneOp = dyn_cast<BinaryOperator>(DefX2->getOperand(1));
387  }
388  if (!SubOneOp)
389  return false;
390 
391  Instruction *SubInst = cast<Instruction>(SubOneOp);
392  ConstantInt *Dec = dyn_cast<ConstantInt>(SubInst->getOperand(1));
393  if (!Dec ||
394  !((SubInst->getOpcode() == Instruction::Sub && Dec->isOne()) ||
395  (SubInst->getOpcode() == Instruction::Add && Dec->isAllOnesValue()))) {
396  return false;
397  }
398  }
399 
400  // step 3: Check the recurrence of variable X
401  {
402  PhiX = dyn_cast<PHINode>(VarX1);
403  if (!PhiX ||
404  (PhiX->getOperand(0) != DefX2 && PhiX->getOperand(1) != DefX2)) {
405  return false;
406  }
407  }
408 
409  // step 4: Find the instruction which count the population: cnt2 = cnt1 + 1
410  {
411  CountInst = nullptr;
412  for (BasicBlock::iterator Iter = LoopEntry->getFirstNonPHI(),
413  IterE = LoopEntry->end(); Iter != IterE; Iter++) {
414  Instruction *Inst = Iter;
415  if (Inst->getOpcode() != Instruction::Add)
416  continue;
417 
418  ConstantInt *Inc = dyn_cast<ConstantInt>(Inst->getOperand(1));
419  if (!Inc || !Inc->isOne())
420  continue;
421 
422  PHINode *Phi = dyn_cast<PHINode>(Inst->getOperand(0));
423  if (!Phi || Phi->getParent() != LoopEntry)
424  continue;
425 
426  // Check if the result of the instruction is live of the loop.
427  bool LiveOutLoop = false;
428  for (User *U : Inst->users()) {
429  if ((cast<Instruction>(U))->getParent() != LoopEntry) {
430  LiveOutLoop = true; break;
431  }
432  }
433 
434  if (LiveOutLoop) {
435  CountInst = Inst;
436  CountPhi = Phi;
437  break;
438  }
439  }
440 
441  if (!CountInst)
442  return false;
443  }
444 
445  // step 5: check if the precondition is in this form:
446  // "if (x != 0) goto loop-head ; else goto somewhere-we-don't-care;"
447  {
448  BranchInst *PreCondBr = LIRUtil::getBranch(PreCondBB);
449  Value *T = matchCondition (PreCondBr, CurLoop->getLoopPreheader());
450  if (T != PhiX->getOperand(0) && T != PhiX->getOperand(1))
451  return false;
452 
453  CntInst = CountInst;
454  CntPhi = CountPhi;
455  Var = T;
456  }
457 
458  return true;
459 }
460 
461 void NclPopcountRecognize::transform(Instruction *CntInst,
462  PHINode *CntPhi, Value *Var) {
463 
464  ScalarEvolution *SE = LIR.getScalarEvolution();
465  TargetLibraryInfo *TLI = LIR.getTargetLibraryInfo();
466  BasicBlock *PreHead = CurLoop->getLoopPreheader();
467  BranchInst *PreCondBr = LIRUtil::getBranch(PreCondBB);
468  const DebugLoc DL = CntInst->getDebugLoc();
469 
470  // Assuming before transformation, the loop is following:
471  // if (x) // the precondition
472  // do { cnt++; x &= x - 1; } while(x);
473 
474  // Step 1: Insert the ctpop instruction at the end of the precondition block
475  IRBuilderTy Builder(PreCondBr);
476  Value *PopCnt, *PopCntZext, *NewCount, *TripCnt;
477  {
478  PopCnt = createPopcntIntrinsic(Builder, Var, DL);
479  NewCount = PopCntZext =
480  Builder.CreateZExtOrTrunc(PopCnt, cast<IntegerType>(CntPhi->getType()));
481 
482  if (NewCount != PopCnt)
483  (cast<Instruction>(NewCount))->setDebugLoc(DL);
484 
485  // TripCnt is exactly the number of iterations the loop has
486  TripCnt = NewCount;
487 
488  // If the population counter's initial value is not zero, insert Add Inst.
489  Value *CntInitVal = CntPhi->getIncomingValueForBlock(PreHead);
490  ConstantInt *InitConst = dyn_cast<ConstantInt>(CntInitVal);
491  if (!InitConst || !InitConst->isZero()) {
492  NewCount = Builder.CreateAdd(NewCount, CntInitVal);
493  (cast<Instruction>(NewCount))->setDebugLoc(DL);
494  }
495  }
496 
497  // Step 2: Replace the precondition from "if(x == 0) goto loop-exit" to
498  // "if(NewCount == 0) loop-exit". Withtout this change, the intrinsic
499  // function would be partial dead code, and downstream passes will drag
500  // it back from the precondition block to the preheader.
501  {
502  ICmpInst *PreCond = cast<ICmpInst>(PreCondBr->getCondition());
503 
504  Value *Opnd0 = PopCntZext;
505  Value *Opnd1 = ConstantInt::get(PopCntZext->getType(), 0);
506  if (PreCond->getOperand(0) != Var)
507  std::swap(Opnd0, Opnd1);
508 
509  ICmpInst *NewPreCond =
510  cast<ICmpInst>(Builder.CreateICmp(PreCond->getPredicate(), Opnd0, Opnd1));
511  PreCondBr->setCondition(NewPreCond);
512 
514  }
515 
516  // Step 3: Note that the population count is exactly the trip count of the
517  // loop in question, which enble us to to convert the loop from noncountable
518  // loop into a countable one. The benefit is twofold:
519  //
520  // - If the loop only counts population, the entire loop become dead after
521  // the transformation. It is lots easier to prove a countable loop dead
522  // than to prove a noncountable one. (In some C dialects, a infite loop
523  // isn't dead even if it computes nothing useful. In general, DCE needs
524  // to prove a noncountable loop finite before safely delete it.)
525  //
526  // - If the loop also performs something else, it remains alive.
527  // Since it is transformed to countable form, it can be aggressively
528  // optimized by some optimizations which are in general not applicable
529  // to a noncountable loop.
530  //
531  // After this step, this loop (conceptually) would look like following:
532  // newcnt = __builtin_ctpop(x);
533  // t = newcnt;
534  // if (x)
535  // do { cnt++; x &= x-1; t--) } while (t > 0);
536  BasicBlock *Body = *(CurLoop->block_begin());
537  {
538  BranchInst *LbBr = LIRUtil::getBranch(Body);
539  ICmpInst *LbCond = cast<ICmpInst>(LbBr->getCondition());
540  Type *Ty = TripCnt->getType();
541 
542  PHINode *TcPhi = PHINode::Create(Ty, 2, "tcphi", Body->begin());
543 
544  Builder.SetInsertPoint(LbCond);
545  Value *Opnd1 = cast<Value>(TcPhi);
546  Value *Opnd2 = cast<Value>(ConstantInt::get(Ty, 1));
547  Instruction *TcDec =
548  cast<Instruction>(Builder.CreateSub(Opnd1, Opnd2, "tcdec", false, true));
549 
550  TcPhi->addIncoming(TripCnt, PreHead);
551  TcPhi->addIncoming(TcDec, Body);
552 
553  CmpInst::Predicate Pred = (LbBr->getSuccessor(0) == Body) ?
555  LbCond->setPredicate(Pred);
556  LbCond->setOperand(0, TcDec);
557  LbCond->setOperand(1, cast<Value>(ConstantInt::get(Ty, 0)));
558  }
559 
560  // Step 4: All the references to the original population counter outside
561  // the loop are replaced with the NewCount -- the value returned from
562  // __builtin_ctpop().
563  CntInst->replaceUsesOutsideBlock(NewCount, Body);
564 
565  // step 5: Forget the "non-computable" trip-count SCEV associated with the
566  // loop. The loop would otherwise not be deleted even if it becomes empty.
567  SE->forgetLoop(CurLoop);
568 }
569 
570 CallInst *NclPopcountRecognize::createPopcntIntrinsic(IRBuilderTy &IRBuilder,
571  Value *Val, DebugLoc DL) {
572  Value *Ops[] = { Val };
573  Type *Tys[] = { Val->getType() };
574 
575  Module *M = (*(CurLoop->block_begin()))->getParent()->getParent();
576  Value *Func = Intrinsic::getDeclaration(M, Intrinsic::ctpop, Tys);
577  CallInst *CI = IRBuilder.CreateCall(Func, Ops);
578  CI->setDebugLoc(DL);
579 
580  return CI;
581 }
582 
583 /// recognize - detect population count idiom in a non-countable loop. If
584 /// detected, transform the relevant code to popcount intrinsic function
585 /// call, and return true; otherwise, return false.
586 bool NclPopcountRecognize::recognize() {
587 
588  if (!LIR.getTargetTransformInfo())
589  return false;
590 
591  LIR.getScalarEvolution();
592 
593  if (!preliminaryScreen())
594  return false;
595 
596  Instruction *CntInst;
597  PHINode *CntPhi;
598  Value *Val;
599  if (!detectIdiom(CntInst, CntPhi, Val))
600  return false;
601 
602  transform(CntInst, CntPhi, Val);
603  return true;
604 }
605 
606 //===----------------------------------------------------------------------===//
607 //
608 // Implementation of LoopIdiomRecognize
609 //
610 //===----------------------------------------------------------------------===//
611 
612 bool LoopIdiomRecognize::runOnCountableLoop() {
613  const SCEV *BECount = SE->getBackedgeTakenCount(CurLoop);
614  assert(!isa<SCEVCouldNotCompute>(BECount) &&
615  "runOnCountableLoop() called on a loop without a predictable"
616  "backedge-taken count");
617 
618  // If this loop executes exactly one time, then it should be peeled, not
619  // optimized by this pass.
620  if (const SCEVConstant *BECst = dyn_cast<SCEVConstant>(BECount))
621  if (BECst->getValue()->getValue() == 0)
622  return false;
623 
624  // set DT
625  (void)getDominatorTree();
626 
627  LoopInfo &LI = getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
628  TLI = &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
629 
630  // set TLI
631  (void)getTargetLibraryInfo();
632 
633  SmallVector<BasicBlock*, 8> ExitBlocks;
634  CurLoop->getUniqueExitBlocks(ExitBlocks);
635 
636  DEBUG(dbgs() << "loop-idiom Scanning: F["
637  << CurLoop->getHeader()->getParent()->getName()
638  << "] Loop %" << CurLoop->getHeader()->getName() << "\n");
639 
640  bool MadeChange = false;
641  // Scan all the blocks in the loop that are not in subloops.
642  for (auto *BB : CurLoop->getBlocks()) {
643  // Ignore blocks in subloops.
644  if (LI.getLoopFor(BB) != CurLoop)
645  continue;
646 
647  MadeChange |= runOnLoopBlock(BB, BECount, ExitBlocks);
648  }
649  return MadeChange;
650 }
651 
652 bool LoopIdiomRecognize::runOnNoncountableLoop() {
653  NclPopcountRecognize Popcount(*this);
654  if (Popcount.recognize())
655  return true;
656 
657  return false;
658 }
659 
660 bool LoopIdiomRecognize::runOnLoop(Loop *L, LPPassManager &LPM) {
661  if (skipOptnoneFunction(L))
662  return false;
663 
664  CurLoop = L;
665 
666  // If the loop could not be converted to canonical form, it must have an
667  // indirectbr in it, just give up.
668  if (!L->getLoopPreheader())
669  return false;
670 
671  // Disable loop idiom recognition if the function's name is a common idiom.
672  StringRef Name = L->getHeader()->getParent()->getName();
673  if (Name == "memset" || Name == "memcpy")
674  return false;
675 
676  SE = &getAnalysis<ScalarEvolution>();
678  return runOnCountableLoop();
679  return runOnNoncountableLoop();
680 }
681 
682 /// runOnLoopBlock - Process the specified block, which lives in a counted loop
683 /// with the specified backedge count. This block is known to be in the current
684 /// loop and not in any subloops.
685 bool LoopIdiomRecognize::runOnLoopBlock(BasicBlock *BB, const SCEV *BECount,
686  SmallVectorImpl<BasicBlock*> &ExitBlocks) {
687  // We can only promote stores in this block if they are unconditionally
688  // executed in the loop. For a block to be unconditionally executed, it has
689  // to dominate all the exit blocks of the loop. Verify this now.
690  for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i)
691  if (!DT->dominates(BB, ExitBlocks[i]))
692  return false;
693 
694  bool MadeChange = false;
695  for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ) {
696  Instruction *Inst = I++;
697  // Look for store instructions, which may be optimized to memset/memcpy.
698  if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
699  WeakVH InstPtr(I);
700  if (!processLoopStore(SI, BECount)) continue;
701  MadeChange = true;
702 
703  // If processing the store invalidated our iterator, start over from the
704  // top of the block.
705  if (!InstPtr)
706  I = BB->begin();
707  continue;
708  }
709 
710  // Look for memset instructions, which may be optimized to a larger memset.
711  if (MemSetInst *MSI = dyn_cast<MemSetInst>(Inst)) {
712  WeakVH InstPtr(I);
713  if (!processLoopMemSet(MSI, BECount)) continue;
714  MadeChange = true;
715 
716  // If processing the memset invalidated our iterator, start over from the
717  // top of the block.
718  if (!InstPtr)
719  I = BB->begin();
720  continue;
721  }
722  }
723 
724  return MadeChange;
725 }
726 
727 
728 /// processLoopStore - See if this store can be promoted to a memset or memcpy.
729 bool LoopIdiomRecognize::processLoopStore(StoreInst *SI, const SCEV *BECount) {
730  if (!SI->isSimple()) return false;
731 
732  Value *StoredVal = SI->getValueOperand();
733  Value *StorePtr = SI->getPointerOperand();
734 
735  // Reject stores that are so large that they overflow an unsigned.
736  auto &DL = CurLoop->getHeader()->getModule()->getDataLayout();
737  uint64_t SizeInBits = DL.getTypeSizeInBits(StoredVal->getType());
738  if ((SizeInBits & 7) || (SizeInBits >> 32) != 0)
739  return false;
740 
741  // See if the pointer expression is an AddRec like {base,+,1} on the current
742  // loop, which indicates a strided store. If we have something else, it's a
743  // random store we can't handle.
744  const SCEVAddRecExpr *StoreEv =
745  dyn_cast<SCEVAddRecExpr>(SE->getSCEV(StorePtr));
746  if (!StoreEv || StoreEv->getLoop() != CurLoop || !StoreEv->isAffine())
747  return false;
748 
749  // Check to see if the stride matches the size of the store. If so, then we
750  // know that every byte is touched in the loop.
751  unsigned StoreSize = (unsigned)SizeInBits >> 3;
752  const SCEVConstant *Stride = dyn_cast<SCEVConstant>(StoreEv->getOperand(1));
753 
754  if (!Stride || StoreSize != Stride->getValue()->getValue()) {
755  // TODO: Could also handle negative stride here someday, that will require
756  // the validity check in mayLoopAccessLocation to be updated though.
757  // Enable this to print exact negative strides.
758  if (0 && Stride && StoreSize == -Stride->getValue()->getValue()) {
759  dbgs() << "NEGATIVE STRIDE: " << *SI << "\n";
760  dbgs() << "BB: " << *SI->getParent();
761  }
762 
763  return false;
764  }
765 
766  // See if we can optimize just this store in isolation.
767  if (processLoopStridedStore(StorePtr, StoreSize, SI->getAlignment(),
768  StoredVal, SI, StoreEv, BECount))
769  return true;
770 
771  // If the stored value is a strided load in the same loop with the same stride
772  // this this may be transformable into a memcpy. This kicks in for stuff like
773  // for (i) A[i] = B[i];
774  if (LoadInst *LI = dyn_cast<LoadInst>(StoredVal)) {
775  const SCEVAddRecExpr *LoadEv =
776  dyn_cast<SCEVAddRecExpr>(SE->getSCEV(LI->getOperand(0)));
777  if (LoadEv && LoadEv->getLoop() == CurLoop && LoadEv->isAffine() &&
778  StoreEv->getOperand(1) == LoadEv->getOperand(1) && LI->isSimple())
779  if (processLoopStoreOfLoopLoad(SI, StoreSize, StoreEv, LoadEv, BECount))
780  return true;
781  }
782  //errs() << "UNHANDLED strided store: " << *StoreEv << " - " << *SI << "\n";
783 
784  return false;
785 }
786 
787 /// processLoopMemSet - See if this memset can be promoted to a large memset.
788 bool LoopIdiomRecognize::
789 processLoopMemSet(MemSetInst *MSI, const SCEV *BECount) {
790  // We can only handle non-volatile memsets with a constant size.
791  if (MSI->isVolatile() || !isa<ConstantInt>(MSI->getLength())) return false;
792 
793  // If we're not allowed to hack on memset, we fail.
794  if (!TLI->has(LibFunc::memset))
795  return false;
796 
797  Value *Pointer = MSI->getDest();
798 
799  // See if the pointer expression is an AddRec like {base,+,1} on the current
800  // loop, which indicates a strided store. If we have something else, it's a
801  // random store we can't handle.
802  const SCEVAddRecExpr *Ev = dyn_cast<SCEVAddRecExpr>(SE->getSCEV(Pointer));
803  if (!Ev || Ev->getLoop() != CurLoop || !Ev->isAffine())
804  return false;
805 
806  // Reject memsets that are so large that they overflow an unsigned.
807  uint64_t SizeInBytes = cast<ConstantInt>(MSI->getLength())->getZExtValue();
808  if ((SizeInBytes >> 32) != 0)
809  return false;
810 
811  // Check to see if the stride matches the size of the memset. If so, then we
812  // know that every byte is touched in the loop.
813  const SCEVConstant *Stride = dyn_cast<SCEVConstant>(Ev->getOperand(1));
814 
815  // TODO: Could also handle negative stride here someday, that will require the
816  // validity check in mayLoopAccessLocation to be updated though.
817  if (!Stride || MSI->getLength() != Stride->getValue())
818  return false;
819 
820  return processLoopStridedStore(Pointer, (unsigned)SizeInBytes,
821  MSI->getAlignment(), MSI->getValue(),
822  MSI, Ev, BECount);
823 }
824 
825 
826 /// mayLoopAccessLocation - Return true if the specified loop might access the
827 /// specified pointer location, which is a loop-strided access. The 'Access'
828 /// argument specifies what the verboten forms of access are (read or write).
830  Loop *L, const SCEV *BECount,
831  unsigned StoreSize, AliasAnalysis &AA,
832  Instruction *IgnoredStore) {
833  // Get the location that may be stored across the loop. Since the access is
834  // strided positively through memory, we say that the modified location starts
835  // at the pointer and has infinite size.
836  uint64_t AccessSize = MemoryLocation::UnknownSize;
837 
838  // If the loop iterates a fixed number of times, we can refine the access size
839  // to be exactly the size of the memset, which is (BECount+1)*StoreSize
840  if (const SCEVConstant *BECst = dyn_cast<SCEVConstant>(BECount))
841  AccessSize = (BECst->getValue()->getZExtValue()+1)*StoreSize;
842 
843  // TODO: For this to be really effective, we have to dive into the pointer
844  // operand in the store. Store to &A[i] of 100 will always return may alias
845  // with store of &A[100], we need to StoreLoc to be "A" with size of 100,
846  // which will then no-alias a store to &A[100].
847  MemoryLocation StoreLoc(Ptr, AccessSize);
848 
849  for (Loop::block_iterator BI = L->block_begin(), E = L->block_end(); BI != E;
850  ++BI)
851  for (BasicBlock::iterator I = (*BI)->begin(), E = (*BI)->end(); I != E; ++I)
852  if (&*I != IgnoredStore &&
853  (AA.getModRefInfo(I, StoreLoc) & Access))
854  return true;
855 
856  return false;
857 }
858 
859 /// getMemSetPatternValue - If a strided store of the specified value is safe to
860 /// turn into a memset_pattern16, return a ConstantArray of 16 bytes that should
861 /// be passed in. Otherwise, return null.
862 ///
863 /// Note that we don't ever attempt to use memset_pattern8 or 4, because these
864 /// just replicate their input array and then pass on to memset_pattern16.
866  // If the value isn't a constant, we can't promote it to being in a constant
867  // array. We could theoretically do a store to an alloca or something, but
868  // that doesn't seem worthwhile.
869  Constant *C = dyn_cast<Constant>(V);
870  if (!C) return nullptr;
871 
872  // Only handle simple values that are a power of two bytes in size.
873  uint64_t Size = DL.getTypeSizeInBits(V->getType());
874  if (Size == 0 || (Size & 7) || (Size & (Size-1)))
875  return nullptr;
876 
877  // Don't care enough about darwin/ppc to implement this.
878  if (DL.isBigEndian())
879  return nullptr;
880 
881  // Convert to size in bytes.
882  Size /= 8;
883 
884  // TODO: If CI is larger than 16-bytes, we can try slicing it in half to see
885  // if the top and bottom are the same (e.g. for vectors and large integers).
886  if (Size > 16) return nullptr;
887 
888  // If the constant is exactly 16 bytes, just use it.
889  if (Size == 16) return C;
890 
891  // Otherwise, we'll use an array of the constants.
892  unsigned ArraySize = 16/Size;
893  ArrayType *AT = ArrayType::get(V->getType(), ArraySize);
894  return ConstantArray::get(AT, std::vector<Constant*>(ArraySize, C));
895 }
896 
897 
898 /// processLoopStridedStore - We see a strided store of some value. If we can
899 /// transform this into a memset or memset_pattern in the loop preheader, do so.
900 bool LoopIdiomRecognize::
901 processLoopStridedStore(Value *DestPtr, unsigned StoreSize,
902  unsigned StoreAlignment, Value *StoredVal,
903  Instruction *TheStore, const SCEVAddRecExpr *Ev,
904  const SCEV *BECount) {
905 
906  // If the stored value is a byte-wise value (like i32 -1), then it may be
907  // turned into a memset of i8 -1, assuming that all the consecutive bytes
908  // are stored. A store of i32 0x01020304 can never be turned into a memset,
909  // but it can be turned into memset_pattern if the target supports it.
910  Value *SplatValue = isBytewiseValue(StoredVal);
911  Constant *PatternValue = nullptr;
912  auto &DL = CurLoop->getHeader()->getModule()->getDataLayout();
913  unsigned DestAS = DestPtr->getType()->getPointerAddressSpace();
914 
915  // If we're allowed to form a memset, and the stored value would be acceptable
916  // for memset, use it.
917  if (SplatValue && TLI->has(LibFunc::memset) &&
918  // Verify that the stored value is loop invariant. If not, we can't
919  // promote the memset.
920  CurLoop->isLoopInvariant(SplatValue)) {
921  // Keep and use SplatValue.
922  PatternValue = nullptr;
923  } else if (DestAS == 0 && TLI->has(LibFunc::memset_pattern16) &&
924  (PatternValue = getMemSetPatternValue(StoredVal, DL))) {
925  // Don't create memset_pattern16s with address spaces.
926  // It looks like we can use PatternValue!
927  SplatValue = nullptr;
928  } else {
929  // Otherwise, this isn't an idiom we can transform. For example, we can't
930  // do anything with a 3-byte store.
931  return false;
932  }
933 
934  // The trip count of the loop and the base pointer of the addrec SCEV is
935  // guaranteed to be loop invariant, which means that it should dominate the
936  // header. This allows us to insert code for it in the preheader.
937  BasicBlock *Preheader = CurLoop->getLoopPreheader();
938  IRBuilder<> Builder(Preheader->getTerminator());
939  SCEVExpander Expander(*SE, DL, "loop-idiom");
940 
941  Type *DestInt8PtrTy = Builder.getInt8PtrTy(DestAS);
942 
943  // Okay, we have a strided store "p[i]" of a splattable value. We can turn
944  // this into a memset in the loop preheader now if we want. However, this
945  // would be unsafe to do if there is anything else in the loop that may read
946  // or write to the aliased location. Check for any overlap by generating the
947  // base pointer and checking the region.
948  Value *BasePtr =
949  Expander.expandCodeFor(Ev->getStart(), DestInt8PtrTy,
950  Preheader->getTerminator());
951 
952  if (mayLoopAccessLocation(BasePtr, AliasAnalysis::ModRef,
953  CurLoop, BECount,
954  StoreSize, getAnalysis<AliasAnalysis>(), TheStore)) {
955  Expander.clear();
956  // If we generated new code for the base pointer, clean up.
958  return false;
959  }
960 
961  // Okay, everything looks good, insert the memset.
962 
963  // The # stored bytes is (BECount+1)*Size. Expand the trip count out to
964  // pointer size if it isn't already.
965  Type *IntPtr = Builder.getIntPtrTy(DL, DestAS);
966  BECount = SE->getTruncateOrZeroExtend(BECount, IntPtr);
967 
968  const SCEV *NumBytesS = SE->getAddExpr(BECount, SE->getConstant(IntPtr, 1),
969  SCEV::FlagNUW);
970  if (StoreSize != 1) {
971  NumBytesS = SE->getMulExpr(NumBytesS, SE->getConstant(IntPtr, StoreSize),
972  SCEV::FlagNUW);
973  }
974 
975  Value *NumBytes =
976  Expander.expandCodeFor(NumBytesS, IntPtr, Preheader->getTerminator());
977 
978  CallInst *NewCall;
979  if (SplatValue) {
980  NewCall = Builder.CreateMemSet(BasePtr,
981  SplatValue,
982  NumBytes,
983  StoreAlignment);
984  } else {
985  // Everything is emitted in default address space
986  Type *Int8PtrTy = DestInt8PtrTy;
987 
988  Module *M = TheStore->getParent()->getParent()->getParent();
989  Value *MSP = M->getOrInsertFunction("memset_pattern16",
990  Builder.getVoidTy(),
991  Int8PtrTy,
992  Int8PtrTy,
993  IntPtr,
994  (void*)nullptr);
995 
996  // Otherwise we should form a memset_pattern16. PatternValue is known to be
997  // an constant array of 16-bytes. Plop the value into a mergable global.
998  GlobalVariable *GV = new GlobalVariable(*M, PatternValue->getType(), true,
999  GlobalValue::PrivateLinkage,
1000  PatternValue, ".memset_pattern");
1001  GV->setUnnamedAddr(true); // Ok to merge these.
1002  GV->setAlignment(16);
1003  Value *PatternPtr = ConstantExpr::getBitCast(GV, Int8PtrTy);
1004  NewCall = Builder.CreateCall(MSP, {BasePtr, PatternPtr, NumBytes});
1005  }
1006 
1007  DEBUG(dbgs() << " Formed memset: " << *NewCall << "\n"
1008  << " from store to: " << *Ev << " at: " << *TheStore << "\n");
1009  NewCall->setDebugLoc(TheStore->getDebugLoc());
1010 
1011  // Okay, the memset has been formed. Zap the original store and anything that
1012  // feeds into it.
1013  deleteDeadInstruction(TheStore, TLI);
1014  ++NumMemSet;
1015  return true;
1016 }
1017 
1018 /// processLoopStoreOfLoopLoad - We see a strided store whose value is a
1019 /// same-strided load.
1020 bool LoopIdiomRecognize::
1021 processLoopStoreOfLoopLoad(StoreInst *SI, unsigned StoreSize,
1022  const SCEVAddRecExpr *StoreEv,
1023  const SCEVAddRecExpr *LoadEv,
1024  const SCEV *BECount) {
1025  // If we're not allowed to form memcpy, we fail.
1026  if (!TLI->has(LibFunc::memcpy))
1027  return false;
1028 
1029  LoadInst *LI = cast<LoadInst>(SI->getValueOperand());
1030 
1031  // The trip count of the loop and the base pointer of the addrec SCEV is
1032  // guaranteed to be loop invariant, which means that it should dominate the
1033  // header. This allows us to insert code for it in the preheader.
1034  BasicBlock *Preheader = CurLoop->getLoopPreheader();
1035  IRBuilder<> Builder(Preheader->getTerminator());
1036  const DataLayout &DL = Preheader->getModule()->getDataLayout();
1037  SCEVExpander Expander(*SE, DL, "loop-idiom");
1038 
1039  // Okay, we have a strided store "p[i]" of a loaded value. We can turn
1040  // this into a memcpy in the loop preheader now if we want. However, this
1041  // would be unsafe to do if there is anything else in the loop that may read
1042  // or write the memory region we're storing to. This includes the load that
1043  // feeds the stores. Check for an alias by generating the base address and
1044  // checking everything.
1045  Value *StoreBasePtr =
1046  Expander.expandCodeFor(StoreEv->getStart(),
1047  Builder.getInt8PtrTy(SI->getPointerAddressSpace()),
1048  Preheader->getTerminator());
1049 
1050  if (mayLoopAccessLocation(StoreBasePtr, AliasAnalysis::ModRef,
1051  CurLoop, BECount, StoreSize,
1052  getAnalysis<AliasAnalysis>(), SI)) {
1053  Expander.clear();
1054  // If we generated new code for the base pointer, clean up.
1055  RecursivelyDeleteTriviallyDeadInstructions(StoreBasePtr, TLI);
1056  return false;
1057  }
1058 
1059  // For a memcpy, we have to make sure that the input array is not being
1060  // mutated by the loop.
1061  Value *LoadBasePtr =
1062  Expander.expandCodeFor(LoadEv->getStart(),
1063  Builder.getInt8PtrTy(LI->getPointerAddressSpace()),
1064  Preheader->getTerminator());
1065 
1066  if (mayLoopAccessLocation(LoadBasePtr, AliasAnalysis::Mod, CurLoop, BECount,
1067  StoreSize, getAnalysis<AliasAnalysis>(), SI)) {
1068  Expander.clear();
1069  // If we generated new code for the base pointer, clean up.
1071  RecursivelyDeleteTriviallyDeadInstructions(StoreBasePtr, TLI);
1072  return false;
1073  }
1074 
1075  // Okay, everything is safe, we can transform this!
1076 
1077 
1078  // The # stored bytes is (BECount+1)*Size. Expand the trip count out to
1079  // pointer size if it isn't already.
1080  Type *IntPtrTy = Builder.getIntPtrTy(DL, SI->getPointerAddressSpace());
1081  BECount = SE->getTruncateOrZeroExtend(BECount, IntPtrTy);
1082 
1083  const SCEV *NumBytesS = SE->getAddExpr(BECount, SE->getConstant(IntPtrTy, 1),
1084  SCEV::FlagNUW);
1085  if (StoreSize != 1)
1086  NumBytesS = SE->getMulExpr(NumBytesS, SE->getConstant(IntPtrTy, StoreSize),
1087  SCEV::FlagNUW);
1088 
1089  Value *NumBytes =
1090  Expander.expandCodeFor(NumBytesS, IntPtrTy, Preheader->getTerminator());
1091 
1092  CallInst *NewCall =
1093  Builder.CreateMemCpy(StoreBasePtr, LoadBasePtr, NumBytes,
1094  std::min(SI->getAlignment(), LI->getAlignment()));
1095  NewCall->setDebugLoc(SI->getDebugLoc());
1096 
1097  DEBUG(dbgs() << " Formed memcpy: " << *NewCall << "\n"
1098  << " from load ptr=" << *LoadEv << " at: " << *LI << "\n"
1099  << " from store ptr=" << *StoreEv << " at: " << *SI << "\n");
1100 
1101 
1102  // Okay, the memset has been formed. Zap the original store and anything that
1103  // feeds into it.
1104  deleteDeadInstruction(SI, TLI);
1105  ++NumMemCpy;
1106  return true;
1107 }
unsigned getAlignment() const
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:82
PopcntSupportKind getPopcntSupport(unsigned IntTyWidthInBit) const
Return hardware support for population count.
Value * getValueOperand()
Definition: Instructions.h:406
iplist< Instruction >::iterator eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing basic block and deletes it...
Definition: Instruction.cpp:70
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:104
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
void addIncoming(Value *V, BasicBlock *BB)
addIncoming - Add an incoming value to the end of the PHI list
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
const SCEV * getConstant(ConstantInt *V)
STATISTIC(NumFunctions,"Total number of functions")
Value * isBytewiseValue(Value *V)
isBytewiseValue - If the specified value can be set by repeating the same byte in memory...
bool isVolatile() const
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:114
unsigned greater than
static void deleteDeadInstruction(Instruction *I, const TargetLibraryInfo *TLI)
deleteDeadInstruction - Delete this instruction.
Value * getValue() const
get* - Return the arguments to the instruction.
value_op_iterator value_op_begin()
Definition: User.h:209
ScalarEvolution - This class is the main scalar evolution driver.
bool isSimple() const
Definition: Instructions.h:401
CallInst - This class represents a function call, abstracting a target machine's calling convention...
MemSetInst - This class wraps the llvm.memset intrinsic.
value_op_iterator value_op_end()
Definition: User.h:212
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:111
A debug info location.
Definition: DebugLoc.h:34
LoadInst - an instruction for reading from memory.
Definition: Instructions.h:177
LoopT * getLoopFor(const BlockT *BB) const
getLoopFor - Return the inner most loop that BB lives in.
Definition: LoopInfo.h:540
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: Type.cpp:216
void setAlignment(unsigned Align)
Definition: Globals.cpp:77
BlockT * getHeader() const
Definition: LoopInfo.h:96
const SCEV * getStart() const
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:231
static bool mayLoopAccessLocation(Value *Ptr, AliasAnalysis::ModRefResult Access, Loop *L, const SCEV *BECount, unsigned StoreSize, AliasAnalysis &AA, Instruction *IgnoredStore)
mayLoopAccessLocation - Return true if the specified loop might access the specified pointer location...
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:70
const APInt & getValue() const
Return the constant as an APInt value reference.
Definition: Constants.h:106
bool has(LibFunc::Func F) const
Tests whether a library function is available.
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:75
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APInt.h:33
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:165
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:517
Number of individual test Apply this number of consecutive mutations to each input exit after the first new interesting input is found the minimized corpus is saved into the first input directory Number of jobs to run If min(jobs, NumberOfCpuCores()/2)\" is used.") FUZZER_FLAG_INT(reload
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Definition: Instructions.h:414
SCEVAddRecExpr - This node represents a polynomial recurrence on the trip count of the specified loop...
Value handle that is nullable, but tries to track the Value.
Definition: ValueHandle.h:141
#define T
ArrayType - Class to represent array types.
Definition: DerivedTypes.h:336
BasicBlock * getSuccessor(unsigned i) const
AnalysisUsage & addPreservedID(const void *ID)
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:866
StoreInst - an instruction for storing to memory.
Definition: Instructions.h:316
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:351
static Constant * getMemSetPatternValue(Value *V, const DataLayout &DL)
getMemSetPatternValue - If a strided store of the specified value is safe to turn into a memset_patte...
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:67
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
Definition: Instructions.h:365
bool isAffine() const
isAffine - Return true if this represents an expression A + B*x where A and B are loop invariant valu...
Wrapper pass for TargetTransformInfo.
void setDebugLoc(DebugLoc Loc)
setDebugLoc - Set the debug location information for this instruction.
Definition: Instruction.h:227
BlockT * getLoopPreheader() const
getLoopPreheader - If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:108
Constant * getOrInsertFunction(StringRef Name, FunctionType *T, AttributeSet AttributeList)
Look up the specified function in the module symbol table.
Definition: Module.cpp:115
LLVM Basic Block Representation.
Definition: BasicBlock.h:65
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
BranchInst - Conditional or Unconditional Branch instruction.
This is an important base class in LLVM.
Definition: Constant.h:41
const SCEV * getOperand(unsigned i) const
char & LCSSAID
Definition: LCSSA.cpp:312
const DebugLoc & getDebugLoc() const
getDebugLoc - Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:230
Represent the analysis usage information of a pass.
This instruction compares its operands according to the predicate given to the constructor.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:697
Value * getOperand(unsigned i) const
Definition: User.h:118
SI Fold Operands
Predicate getPredicate() const
Return the predicate for this instruction.
Definition: InstrTypes.h:760
#define INITIALIZE_AG_DEPENDENCY(depName)
Definition: PassSupport.h:72
static UndefValue * get(Type *T)
get() - Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1473
bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr)
RecursivelyDeleteTriviallyDeadInstructions - If the specified value is a trivially dead instruction...
Definition: Local.cpp:340
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:283
char & LoopSimplifyID
bool isConditional() const
Representation for a specific memory location.
loop Recognize loop false
This is the shared class of boolean and integer constants.
Definition: Constants.h:47
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Value * getDest() const
getDest - This is just like getRawDest, but it strips off any cast instructions that feed it...
iterator end()
Definition: BasicBlock.h:233
AnalysisUsage & addRequiredID(const void *ID)
Definition: Pass.cpp:276
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:861
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:222
Provides information about what library functions are available for the current target.
void setUnnamedAddr(bool Val)
Definition: GlobalValue.h:131
ModRefResult getModRefInfo(const Instruction *I)
getModRefInfo - Return information about whether or not an instruction may read or write memory (with...
Value * getLength() const
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition: Constants.h:161
ConstantInt * getValue() const
unsigned getPointerAddressSpace() const
Returns the address space of the pointer operand.
Definition: Instructions.h:289
void setPredicate(Predicate P)
Set the predicate for this instruction to the specified value.
Definition: InstrTypes.h:765
void setOperand(unsigned i, Value *Val)
Definition: User.h:122
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:123
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:576
loop Recognize loop idioms
loop idiom
Value * getIncomingValueForBlock(const BasicBlock *BB) const
iterator_range< user_iterator > users()
Definition: Value.h:300
BasicBlock * getSinglePredecessor()
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:211
This class uses information about analyze scalars to rewrite expressions in canonical form...
LLVM_ATTRIBUTE_UNUSED_RESULT std::enable_if< !is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:285
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
getAddExpr - Get a canonical add expression, or something simpler if possible.
Pass * createLoopIdiomPass()
APInt And(const APInt &LHS, const APInt &RHS)
Bitwise AND function for APInt.
Definition: APInt.h:1890
block_iterator block_end() const
Definition: LoopInfo.h:142
Value * getCondition() const
void forgetLoop(const Loop *L)
forgetLoop - This method should be called by the client when it has changed a loop in a way that may ...
SCEV - This class represents an analyzed expression in the program.
unsigned getAlignment() const
getAlignment - Return the alignment of the access that is being performed
Definition: Instructions.h:243
#define I(x, y, z)
Definition: MD5.cpp:54
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:124
const Loop * getLoop() const
size_t size() const
Definition: BasicBlock.h:241
void initializeLoopIdiomRecognizePass(PassRegistry &)
void setCondition(Value *V)
signed less or equal
const SCEV * getBackedgeTakenCount(const Loop *L)
getBackedgeTakenCount - If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCouldNotCompute object.
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:365
LLVM Value Representation.
Definition: Value.h:69
ModRefResult
Simple mod/ref information...
const SCEV * getSCEV(Value *V)
getSCEV - Return a SCEV expression for the full generality of the specified expression.
unsigned getOpcode() const
getOpcode() returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:112
uint64_t getTypeSizeInBits(Type *Ty) const
Size examples:
Definition: DataLayout.h:507
#define DEBUG(X)
Definition: Debug.h:92
block_iterator block_begin() const
Definition: LoopInfo.h:141
The legacy pass manager's analysis pass to compute loop information.
Definition: LoopInfo.h:737
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:40
void replaceUsesOutsideBlock(Value *V, BasicBlock *BB)
replaceUsesOutsideBlock - Go through the uses list for this definition and make each use point to "V"...
Definition: Value.cpp:384
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:203
const SCEV * getTruncateOrZeroExtend(const SCEV *V, Type *Ty)
getTruncateOrZeroExtend - Return a SCEV corresponding to a conversion of the input value to the speci...
This pass exposes codegen information to IR-level passes.
INITIALIZE_PASS_BEGIN(LoopIdiomRecognize,"loop-idiom","Recognize loop idioms", false, false) INITIALIZE_PASS_END(LoopIdiomRecognize
bool isBigEndian() const
Definition: DataLayout.h:218
const SCEV * getMulExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
getMulExpr - Get a canonical multiply expression, or something simpler if possible.
bool hasLoopInvariantBackedgeTakenCount(const Loop *L)
hasLoopInvariantBackedgeTakenCount - Return true if the specified loop has an analyzable loop-invaria...
Value * getPointerOperand()
Definition: Instructions.h:409
const BasicBlock * getParent() const
Definition: Instruction.h:72
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:169
SCEVConstant - This class represents a constant integer value.