LLVM  4.0.0
LoopUnrollRuntime.cpp
Go to the documentation of this file.
1 //===-- UnrollLoopRuntime.cpp - Runtime Loop unrolling utilities ----------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file implements some loop unrolling utilities for loops with run-time
11 // trip counts. See LoopUnroll.cpp for unrolling loops with compile-time
12 // trip counts.
13 //
14 // The functions in this file are used to generate extra code when the
15 // run-time trip count modulo the unroll factor is not 0. When this is the
16 // case, we need to generate code to execute these 'left over' iterations.
17 //
18 // The current strategy generates an if-then-else sequence prior to the
19 // unrolled loop to execute the 'left over' iterations before or after the
20 // unrolled loop.
21 //
22 //===----------------------------------------------------------------------===//
23 
25 #include "llvm/ADT/Statistic.h"
28 #include "llvm/Analysis/LoopPass.h"
31 #include "llvm/IR/BasicBlock.h"
32 #include "llvm/IR/Dominators.h"
33 #include "llvm/IR/Metadata.h"
34 #include "llvm/IR/Module.h"
35 #include "llvm/Support/Debug.h"
37 #include "llvm/Transforms/Scalar.h"
40 #include <algorithm>
41 
42 using namespace llvm;
43 
44 #define DEBUG_TYPE "loop-unroll"
45 
46 STATISTIC(NumRuntimeUnrolled,
47  "Number of loops unrolled with run-time trip counts");
48 
49 /// Connect the unrolling prolog code to the original loop.
50 /// The unrolling prolog code contains code to execute the
51 /// 'extra' iterations if the run-time trip count modulo the
52 /// unroll count is non-zero.
53 ///
54 /// This function performs the following:
55 /// - Create PHI nodes at prolog end block to combine values
56 /// that exit the prolog code and jump around the prolog.
57 /// - Add a PHI operand to a PHI node at the loop exit block
58 /// for values that exit the prolog and go around the loop.
59 /// - Branch around the original loop if the trip count is less
60 /// than the unroll factor.
61 ///
62 static void ConnectProlog(Loop *L, Value *BECount, unsigned Count,
63  BasicBlock *PrologExit, BasicBlock *PreHeader,
64  BasicBlock *NewPreHeader, ValueToValueMapTy &VMap,
65  DominatorTree *DT, LoopInfo *LI, bool PreserveLCSSA) {
66  BasicBlock *Latch = L->getLoopLatch();
67  assert(Latch && "Loop must have a latch");
68  BasicBlock *PrologLatch = cast<BasicBlock>(VMap[Latch]);
69 
70  // Create a PHI node for each outgoing value from the original loop
71  // (which means it is an outgoing value from the prolog code too).
72  // The new PHI node is inserted in the prolog end basic block.
73  // The new PHI node value is added as an operand of a PHI node in either
74  // the loop header or the loop exit block.
75  for (BasicBlock *Succ : successors(Latch)) {
76  for (Instruction &BBI : *Succ) {
77  PHINode *PN = dyn_cast<PHINode>(&BBI);
78  // Exit when we passed all PHI nodes.
79  if (!PN)
80  break;
81  // Add a new PHI node to the prolog end block and add the
82  // appropriate incoming values.
83  PHINode *NewPN = PHINode::Create(PN->getType(), 2, PN->getName() + ".unr",
84  PrologExit->getFirstNonPHI());
85  // Adding a value to the new PHI node from the original loop preheader.
86  // This is the value that skips all the prolog code.
87  if (L->contains(PN)) {
88  NewPN->addIncoming(PN->getIncomingValueForBlock(NewPreHeader),
89  PreHeader);
90  } else {
91  NewPN->addIncoming(UndefValue::get(PN->getType()), PreHeader);
92  }
93 
94  Value *V = PN->getIncomingValueForBlock(Latch);
95  if (Instruction *I = dyn_cast<Instruction>(V)) {
96  if (L->contains(I)) {
97  V = VMap.lookup(I);
98  }
99  }
100  // Adding a value to the new PHI node from the last prolog block
101  // that was created.
102  NewPN->addIncoming(V, PrologLatch);
103 
104  // Update the existing PHI node operand with the value from the
105  // new PHI node. How this is done depends on if the existing
106  // PHI node is in the original loop block, or the exit block.
107  if (L->contains(PN)) {
108  PN->setIncomingValue(PN->getBasicBlockIndex(NewPreHeader), NewPN);
109  } else {
110  PN->addIncoming(NewPN, PrologExit);
111  }
112  }
113  }
114 
115  // Make sure that created prolog loop is in simplified form
116  SmallVector<BasicBlock *, 4> PrologExitPreds;
117  Loop *PrologLoop = LI->getLoopFor(PrologLatch);
118  if (PrologLoop) {
119  for (BasicBlock *PredBB : predecessors(PrologExit))
120  if (PrologLoop->contains(PredBB))
121  PrologExitPreds.push_back(PredBB);
122 
123  SplitBlockPredecessors(PrologExit, PrologExitPreds, ".unr-lcssa", DT, LI,
124  PreserveLCSSA);
125  }
126 
127  // Create a branch around the original loop, which is taken if there are no
128  // iterations remaining to be executed after running the prologue.
129  Instruction *InsertPt = PrologExit->getTerminator();
130  IRBuilder<> B(InsertPt);
131 
132  assert(Count != 0 && "nonsensical Count!");
133 
134  // If BECount <u (Count - 1) then (BECount + 1) % Count == (BECount + 1)
135  // This means %xtraiter is (BECount + 1) and all of the iterations of this
136  // loop were executed by the prologue. Note that if BECount <u (Count - 1)
137  // then (BECount + 1) cannot unsigned-overflow.
138  Value *BrLoopExit =
139  B.CreateICmpULT(BECount, ConstantInt::get(BECount->getType(), Count - 1));
140  BasicBlock *Exit = L->getUniqueExitBlock();
141  assert(Exit && "Loop must have a single exit block only");
142  // Split the exit to maintain loop canonicalization guarantees
144  SplitBlockPredecessors(Exit, Preds, ".unr-lcssa", DT, LI,
145  PreserveLCSSA);
146  // Add the branch to the exit block (around the unrolled loop)
147  B.CreateCondBr(BrLoopExit, Exit, NewPreHeader);
148  InsertPt->eraseFromParent();
149 }
150 
151 /// Connect the unrolling epilog code to the original loop.
152 /// The unrolling epilog code contains code to execute the
153 /// 'extra' iterations if the run-time trip count modulo the
154 /// unroll count is non-zero.
155 ///
156 /// This function performs the following:
157 /// - Update PHI nodes at the unrolling loop exit and epilog loop exit
158 /// - Create PHI nodes at the unrolling loop exit to combine
159 /// values that exit the unrolling loop code and jump around it.
160 /// - Update PHI operands in the epilog loop by the new PHI nodes
161 /// - Branch around the epilog loop if extra iters (ModVal) is zero.
162 ///
163 static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit,
164  BasicBlock *Exit, BasicBlock *PreHeader,
165  BasicBlock *EpilogPreHeader, BasicBlock *NewPreHeader,
166  ValueToValueMapTy &VMap, DominatorTree *DT,
167  LoopInfo *LI, bool PreserveLCSSA) {
168  BasicBlock *Latch = L->getLoopLatch();
169  assert(Latch && "Loop must have a latch");
170  BasicBlock *EpilogLatch = cast<BasicBlock>(VMap[Latch]);
171 
172  // Loop structure should be the following:
173  //
174  // PreHeader
175  // NewPreHeader
176  // Header
177  // ...
178  // Latch
179  // NewExit (PN)
180  // EpilogPreHeader
181  // EpilogHeader
182  // ...
183  // EpilogLatch
184  // Exit (EpilogPN)
185 
186  // Update PHI nodes at NewExit and Exit.
187  for (Instruction &BBI : *NewExit) {
188  PHINode *PN = dyn_cast<PHINode>(&BBI);
189  // Exit when we passed all PHI nodes.
190  if (!PN)
191  break;
192  // PN should be used in another PHI located in Exit block as
193  // Exit was split by SplitBlockPredecessors into Exit and NewExit
194  // Basicaly it should look like:
195  // NewExit:
196  // PN = PHI [I, Latch]
197  // ...
198  // Exit:
199  // EpilogPN = PHI [PN, EpilogPreHeader]
200  //
201  // There is EpilogPreHeader incoming block instead of NewExit as
202  // NewExit was spilt 1 more time to get EpilogPreHeader.
203  assert(PN->hasOneUse() && "The phi should have 1 use");
204  PHINode *EpilogPN = cast<PHINode> (PN->use_begin()->getUser());
205  assert(EpilogPN->getParent() == Exit && "EpilogPN should be in Exit block");
206 
207  // Add incoming PreHeader from branch around the Loop
208  PN->addIncoming(UndefValue::get(PN->getType()), PreHeader);
209 
210  Value *V = PN->getIncomingValueForBlock(Latch);
212  if (I && L->contains(I))
213  // If value comes from an instruction in the loop add VMap value.
214  V = VMap.lookup(I);
215  // For the instruction out of the loop, constant or undefined value
216  // insert value itself.
217  EpilogPN->addIncoming(V, EpilogLatch);
218 
219  assert(EpilogPN->getBasicBlockIndex(EpilogPreHeader) >= 0 &&
220  "EpilogPN should have EpilogPreHeader incoming block");
221  // Change EpilogPreHeader incoming block to NewExit.
222  EpilogPN->setIncomingBlock(EpilogPN->getBasicBlockIndex(EpilogPreHeader),
223  NewExit);
224  // Now PHIs should look like:
225  // NewExit:
226  // PN = PHI [I, Latch], [undef, PreHeader]
227  // ...
228  // Exit:
229  // EpilogPN = PHI [PN, NewExit], [VMap[I], EpilogLatch]
230  }
231 
232  // Create PHI nodes at NewExit (from the unrolling loop Latch and PreHeader).
233  // Update corresponding PHI nodes in epilog loop.
234  for (BasicBlock *Succ : successors(Latch)) {
235  // Skip this as we already updated phis in exit blocks.
236  if (!L->contains(Succ))
237  continue;
238  for (Instruction &BBI : *Succ) {
239  PHINode *PN = dyn_cast<PHINode>(&BBI);
240  // Exit when we passed all PHI nodes.
241  if (!PN)
242  break;
243  // Add new PHI nodes to the loop exit block and update epilog
244  // PHIs with the new PHI values.
245  PHINode *NewPN = PHINode::Create(PN->getType(), 2, PN->getName() + ".unr",
246  NewExit->getFirstNonPHI());
247  // Adding a value to the new PHI node from the unrolling loop preheader.
248  NewPN->addIncoming(PN->getIncomingValueForBlock(NewPreHeader), PreHeader);
249  // Adding a value to the new PHI node from the unrolling loop latch.
250  NewPN->addIncoming(PN->getIncomingValueForBlock(Latch), Latch);
251 
252  // Update the existing PHI node operand with the value from the new PHI
253  // node. Corresponding instruction in epilog loop should be PHI.
254  PHINode *VPN = cast<PHINode>(VMap[&BBI]);
255  VPN->setIncomingValue(VPN->getBasicBlockIndex(EpilogPreHeader), NewPN);
256  }
257  }
258 
259  Instruction *InsertPt = NewExit->getTerminator();
260  IRBuilder<> B(InsertPt);
261  Value *BrLoopExit = B.CreateIsNotNull(ModVal, "lcmp.mod");
262  assert(Exit && "Loop must have a single exit block only");
263  // Split the exit to maintain loop canonicalization guarantees
265  SplitBlockPredecessors(Exit, Preds, ".epilog-lcssa", DT, LI,
266  PreserveLCSSA);
267  // Add the branch to the exit block (around the unrolling loop)
268  B.CreateCondBr(BrLoopExit, EpilogPreHeader, Exit);
269  InsertPt->eraseFromParent();
270 }
271 
272 /// Create a clone of the blocks in a loop and connect them together.
273 /// If CreateRemainderLoop is false, loop structure will not be cloned,
274 /// otherwise a new loop will be created including all cloned blocks, and the
275 /// iterator of it switches to count NewIter down to 0.
276 /// The cloned blocks should be inserted between InsertTop and InsertBot.
277 /// If loop structure is cloned InsertTop should be new preheader, InsertBot
278 /// new loop exit.
279 ///
280 static void CloneLoopBlocks(Loop *L, Value *NewIter,
281  const bool CreateRemainderLoop,
282  const bool UseEpilogRemainder,
283  BasicBlock *InsertTop, BasicBlock *InsertBot,
284  BasicBlock *Preheader,
285  std::vector<BasicBlock *> &NewBlocks,
286  LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap,
287  LoopInfo *LI) {
288  StringRef suffix = UseEpilogRemainder ? "epil" : "prol";
289  BasicBlock *Header = L->getHeader();
290  BasicBlock *Latch = L->getLoopLatch();
291  Function *F = Header->getParent();
292  LoopBlocksDFS::RPOIterator BlockBegin = LoopBlocks.beginRPO();
293  LoopBlocksDFS::RPOIterator BlockEnd = LoopBlocks.endRPO();
294  Loop *NewLoop = nullptr;
295  Loop *ParentLoop = L->getParentLoop();
296  if (CreateRemainderLoop) {
297  NewLoop = new Loop();
298  if (ParentLoop)
299  ParentLoop->addChildLoop(NewLoop);
300  else
301  LI->addTopLevelLoop(NewLoop);
302  }
303 
304  NewLoopsMap NewLoops;
305  if (NewLoop)
306  NewLoops[L] = NewLoop;
307  else if (ParentLoop)
308  NewLoops[L] = ParentLoop;
309 
310  // For each block in the original loop, create a new copy,
311  // and update the value map with the newly created values.
312  for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
313  BasicBlock *NewBB = CloneBasicBlock(*BB, VMap, "." + suffix, F);
314  NewBlocks.push_back(NewBB);
315 
316  // If we're unrolling the outermost loop, there's no remainder loop,
317  // and this block isn't in a nested loop, then the new block is not
318  // in any loop. Otherwise, add it to loopinfo.
319  if (CreateRemainderLoop || LI->getLoopFor(*BB) != L || ParentLoop)
320  addClonedBlockToLoopInfo(*BB, NewBB, LI, NewLoops);
321 
322  VMap[*BB] = NewBB;
323  if (Header == *BB) {
324  // For the first block, add a CFG connection to this newly
325  // created block.
326  InsertTop->getTerminator()->setSuccessor(0, NewBB);
327  }
328 
329  if (Latch == *BB) {
330  // For the last block, if CreateRemainderLoop is false, create a direct
331  // jump to InsertBot. If not, create a loop back to cloned head.
332  VMap.erase((*BB)->getTerminator());
333  BasicBlock *FirstLoopBB = cast<BasicBlock>(VMap[Header]);
334  BranchInst *LatchBR = cast<BranchInst>(NewBB->getTerminator());
335  IRBuilder<> Builder(LatchBR);
336  if (!CreateRemainderLoop) {
337  Builder.CreateBr(InsertBot);
338  } else {
339  PHINode *NewIdx = PHINode::Create(NewIter->getType(), 2,
340  suffix + ".iter",
341  FirstLoopBB->getFirstNonPHI());
342  Value *IdxSub =
343  Builder.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1),
344  NewIdx->getName() + ".sub");
345  Value *IdxCmp =
346  Builder.CreateIsNotNull(IdxSub, NewIdx->getName() + ".cmp");
347  Builder.CreateCondBr(IdxCmp, FirstLoopBB, InsertBot);
348  NewIdx->addIncoming(NewIter, InsertTop);
349  NewIdx->addIncoming(IdxSub, NewBB);
350  }
351  LatchBR->eraseFromParent();
352  }
353  }
354 
355  // Change the incoming values to the ones defined in the preheader or
356  // cloned loop.
357  for (BasicBlock::iterator I = Header->begin(); isa<PHINode>(I); ++I) {
358  PHINode *NewPHI = cast<PHINode>(VMap[&*I]);
359  if (!CreateRemainderLoop) {
360  if (UseEpilogRemainder) {
361  unsigned idx = NewPHI->getBasicBlockIndex(Preheader);
362  NewPHI->setIncomingBlock(idx, InsertTop);
363  NewPHI->removeIncomingValue(Latch, false);
364  } else {
365  VMap[&*I] = NewPHI->getIncomingValueForBlock(Preheader);
366  cast<BasicBlock>(VMap[Header])->getInstList().erase(NewPHI);
367  }
368  } else {
369  unsigned idx = NewPHI->getBasicBlockIndex(Preheader);
370  NewPHI->setIncomingBlock(idx, InsertTop);
371  BasicBlock *NewLatch = cast<BasicBlock>(VMap[Latch]);
372  idx = NewPHI->getBasicBlockIndex(Latch);
373  Value *InVal = NewPHI->getIncomingValue(idx);
374  NewPHI->setIncomingBlock(idx, NewLatch);
375  if (Value *V = VMap.lookup(InVal))
376  NewPHI->setIncomingValue(idx, V);
377  }
378  }
379  if (NewLoop) {
380  // Add unroll disable metadata to disable future unrolling for this loop.
382  // Reserve first location for self reference to the LoopID metadata node.
383  MDs.push_back(nullptr);
384  MDNode *LoopID = NewLoop->getLoopID();
385  if (LoopID) {
386  // First remove any existing loop unrolling metadata.
387  for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) {
388  bool IsUnrollMetadata = false;
389  MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
390  if (MD) {
391  const MDString *S = dyn_cast<MDString>(MD->getOperand(0));
392  IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll.");
393  }
394  if (!IsUnrollMetadata)
395  MDs.push_back(LoopID->getOperand(i));
396  }
397  }
398 
399  LLVMContext &Context = NewLoop->getHeader()->getContext();
400  SmallVector<Metadata *, 1> DisableOperands;
401  DisableOperands.push_back(MDString::get(Context, "llvm.loop.unroll.disable"));
402  MDNode *DisableNode = MDNode::get(Context, DisableOperands);
403  MDs.push_back(DisableNode);
404 
405  MDNode *NewLoopID = MDNode::get(Context, MDs);
406  // Set operand 0 to refer to the loop id itself.
407  NewLoopID->replaceOperandWith(0, NewLoopID);
408  NewLoop->setLoopID(NewLoopID);
409  }
410 }
411 
412 /// Insert code in the prolog/epilog code when unrolling a loop with a
413 /// run-time trip-count.
414 ///
415 /// This method assumes that the loop unroll factor is total number
416 /// of loop bodies in the loop after unrolling. (Some folks refer
417 /// to the unroll factor as the number of *extra* copies added).
418 /// We assume also that the loop unroll factor is a power-of-two. So, after
419 /// unrolling the loop, the number of loop bodies executed is 2,
420 /// 4, 8, etc. Note - LLVM converts the if-then-sequence to a switch
421 /// instruction in SimplifyCFG.cpp. Then, the backend decides how code for
422 /// the switch instruction is generated.
423 ///
424 /// ***Prolog case***
425 /// extraiters = tripcount % loopfactor
426 /// if (extraiters == 0) jump Loop:
427 /// else jump Prol:
428 /// Prol: LoopBody;
429 /// extraiters -= 1 // Omitted if unroll factor is 2.
430 /// if (extraiters != 0) jump Prol: // Omitted if unroll factor is 2.
431 /// if (tripcount < loopfactor) jump End:
432 /// Loop:
433 /// ...
434 /// End:
435 ///
436 /// ***Epilog case***
437 /// extraiters = tripcount % loopfactor
438 /// if (tripcount < loopfactor) jump LoopExit:
439 /// unroll_iters = tripcount - extraiters
440 /// Loop: LoopBody; (executes unroll_iter times);
441 /// unroll_iter -= 1
442 /// if (unroll_iter != 0) jump Loop:
443 /// LoopExit:
444 /// if (extraiters == 0) jump EpilExit:
445 /// Epil: LoopBody; (executes extraiters times)
446 /// extraiters -= 1 // Omitted if unroll factor is 2.
447 /// if (extraiters != 0) jump Epil: // Omitted if unroll factor is 2.
448 /// EpilExit:
449 
451  bool AllowExpensiveTripCount,
452  bool UseEpilogRemainder,
453  LoopInfo *LI, ScalarEvolution *SE,
454  DominatorTree *DT, bool PreserveLCSSA) {
455  // for now, only unroll loops that contain a single exit
456  if (!L->getExitingBlock())
457  return false;
458 
459  // Make sure the loop is in canonical form, and there is a single
460  // exit block only.
461  if (!L->isLoopSimplifyForm())
462  return false;
463  BasicBlock *Exit = L->getUniqueExitBlock(); // successor out of loop
464  if (!Exit)
465  return false;
466 
467  // Use Scalar Evolution to compute the trip count. This allows more loops to
468  // be unrolled than relying on induction var simplification.
469  if (!SE)
470  return false;
471 
472  // Only unroll loops with a computable trip count, and the trip count needs
473  // to be an int value (allowing a pointer type is a TODO item).
474  const SCEV *BECountSC = SE->getBackedgeTakenCount(L);
475  if (isa<SCEVCouldNotCompute>(BECountSC) ||
476  !BECountSC->getType()->isIntegerTy())
477  return false;
478 
479  unsigned BEWidth = cast<IntegerType>(BECountSC->getType())->getBitWidth();
480 
481  // Add 1 since the backedge count doesn't include the first loop iteration.
482  const SCEV *TripCountSC =
483  SE->getAddExpr(BECountSC, SE->getConstant(BECountSC->getType(), 1));
484  if (isa<SCEVCouldNotCompute>(TripCountSC))
485  return false;
486 
487  BasicBlock *Header = L->getHeader();
488  BasicBlock *PreHeader = L->getLoopPreheader();
489  BranchInst *PreHeaderBR = cast<BranchInst>(PreHeader->getTerminator());
490  const DataLayout &DL = Header->getModule()->getDataLayout();
491  SCEVExpander Expander(*SE, DL, "loop-unroll");
492  if (!AllowExpensiveTripCount &&
493  Expander.isHighCostExpansion(TripCountSC, L, PreHeaderBR))
494  return false;
495 
496  // This constraint lets us deal with an overflowing trip count easily; see the
497  // comment on ModVal below.
498  if (Log2_32(Count) > BEWidth)
499  return false;
500 
501  BasicBlock *Latch = L->getLoopLatch();
502 
503  // Loop structure is the following:
504  //
505  // PreHeader
506  // Header
507  // ...
508  // Latch
509  // Exit
510 
511  BasicBlock *NewPreHeader;
512  BasicBlock *NewExit = nullptr;
513  BasicBlock *PrologExit = nullptr;
514  BasicBlock *EpilogPreHeader = nullptr;
515  BasicBlock *PrologPreHeader = nullptr;
516 
517  if (UseEpilogRemainder) {
518  // If epilog remainder
519  // Split PreHeader to insert a branch around loop for unrolling.
520  NewPreHeader = SplitBlock(PreHeader, PreHeader->getTerminator(), DT, LI);
521  NewPreHeader->setName(PreHeader->getName() + ".new");
522  // Split Exit to create phi nodes from branch above.
524  NewExit = SplitBlockPredecessors(Exit, Preds, ".unr-lcssa",
525  DT, LI, PreserveLCSSA);
526  // Split NewExit to insert epilog remainder loop.
527  EpilogPreHeader = SplitBlock(NewExit, NewExit->getTerminator(), DT, LI);
528  EpilogPreHeader->setName(Header->getName() + ".epil.preheader");
529  } else {
530  // If prolog remainder
531  // Split the original preheader twice to insert prolog remainder loop
532  PrologPreHeader = SplitEdge(PreHeader, Header, DT, LI);
533  PrologPreHeader->setName(Header->getName() + ".prol.preheader");
534  PrologExit = SplitBlock(PrologPreHeader, PrologPreHeader->getTerminator(),
535  DT, LI);
536  PrologExit->setName(Header->getName() + ".prol.loopexit");
537  // Split PrologExit to get NewPreHeader.
538  NewPreHeader = SplitBlock(PrologExit, PrologExit->getTerminator(), DT, LI);
539  NewPreHeader->setName(PreHeader->getName() + ".new");
540  }
541  // Loop structure should be the following:
542  // Epilog Prolog
543  //
544  // PreHeader PreHeader
545  // *NewPreHeader *PrologPreHeader
546  // Header *PrologExit
547  // ... *NewPreHeader
548  // Latch Header
549  // *NewExit ...
550  // *EpilogPreHeader Latch
551  // Exit Exit
552 
553  // Calculate conditions for branch around loop for unrolling
554  // in epilog case and around prolog remainder loop in prolog case.
555  // Compute the number of extra iterations required, which is:
556  // extra iterations = run-time trip count % loop unroll factor
557  PreHeaderBR = cast<BranchInst>(PreHeader->getTerminator());
558  Value *TripCount = Expander.expandCodeFor(TripCountSC, TripCountSC->getType(),
559  PreHeaderBR);
560  Value *BECount = Expander.expandCodeFor(BECountSC, BECountSC->getType(),
561  PreHeaderBR);
562  IRBuilder<> B(PreHeaderBR);
563  Value *ModVal;
564  // Calculate ModVal = (BECount + 1) % Count.
565  // Note that TripCount is BECount + 1.
566  if (isPowerOf2_32(Count)) {
567  // When Count is power of 2 we don't BECount for epilog case, however we'll
568  // need it for a branch around unrolling loop for prolog case.
569  ModVal = B.CreateAnd(TripCount, Count - 1, "xtraiter");
570  // 1. There are no iterations to be run in the prolog/epilog loop.
571  // OR
572  // 2. The addition computing TripCount overflowed.
573  //
574  // If (2) is true, we know that TripCount really is (1 << BEWidth) and so
575  // the number of iterations that remain to be run in the original loop is a
576  // multiple Count == (1 << Log2(Count)) because Log2(Count) <= BEWidth (we
577  // explicitly check this above).
578  } else {
579  // As (BECount + 1) can potentially unsigned overflow we count
580  // (BECount % Count) + 1 which is overflow safe as BECount % Count < Count.
581  Value *ModValTmp = B.CreateURem(BECount,
582  ConstantInt::get(BECount->getType(),
583  Count));
584  Value *ModValAdd = B.CreateAdd(ModValTmp,
585  ConstantInt::get(ModValTmp->getType(), 1));
586  // At that point (BECount % Count) + 1 could be equal to Count.
587  // To handle this case we need to take mod by Count one more time.
588  ModVal = B.CreateURem(ModValAdd,
589  ConstantInt::get(BECount->getType(), Count),
590  "xtraiter");
591  }
592  Value *BranchVal =
593  UseEpilogRemainder ? B.CreateICmpULT(BECount,
594  ConstantInt::get(BECount->getType(),
595  Count - 1)) :
596  B.CreateIsNotNull(ModVal, "lcmp.mod");
597  BasicBlock *RemainderLoop = UseEpilogRemainder ? NewExit : PrologPreHeader;
598  BasicBlock *UnrollingLoop = UseEpilogRemainder ? NewPreHeader : PrologExit;
599  // Branch to either remainder (extra iterations) loop or unrolling loop.
600  B.CreateCondBr(BranchVal, RemainderLoop, UnrollingLoop);
601  PreHeaderBR->eraseFromParent();
602  Function *F = Header->getParent();
603  // Get an ordered list of blocks in the loop to help with the ordering of the
604  // cloned blocks in the prolog/epilog code
605  LoopBlocksDFS LoopBlocks(L);
606  LoopBlocks.perform(LI);
607 
608  //
609  // For each extra loop iteration, create a copy of the loop's basic blocks
610  // and generate a condition that branches to the copy depending on the
611  // number of 'left over' iterations.
612  //
613  std::vector<BasicBlock *> NewBlocks;
614  ValueToValueMapTy VMap;
615 
616  // For unroll factor 2 remainder loop will have 1 iterations.
617  // Do not create 1 iteration loop.
618  bool CreateRemainderLoop = (Count != 2);
619 
620  // Clone all the basic blocks in the loop. If Count is 2, we don't clone
621  // the loop, otherwise we create a cloned loop to execute the extra
622  // iterations. This function adds the appropriate CFG connections.
623  BasicBlock *InsertBot = UseEpilogRemainder ? Exit : PrologExit;
624  BasicBlock *InsertTop = UseEpilogRemainder ? EpilogPreHeader : PrologPreHeader;
625  CloneLoopBlocks(L, ModVal, CreateRemainderLoop, UseEpilogRemainder, InsertTop,
626  InsertBot, NewPreHeader, NewBlocks, LoopBlocks, VMap, LI);
627 
628  // Insert the cloned blocks into the function.
629  F->getBasicBlockList().splice(InsertBot->getIterator(),
630  F->getBasicBlockList(),
631  NewBlocks[0]->getIterator(),
632  F->end());
633 
634  // Loop structure should be the following:
635  // Epilog Prolog
636  //
637  // PreHeader PreHeader
638  // NewPreHeader PrologPreHeader
639  // Header PrologHeader
640  // ... ...
641  // Latch PrologLatch
642  // NewExit PrologExit
643  // EpilogPreHeader NewPreHeader
644  // EpilogHeader Header
645  // ... ...
646  // EpilogLatch Latch
647  // Exit Exit
648 
649  // Rewrite the cloned instruction operands to use the values created when the
650  // clone is created.
651  for (BasicBlock *BB : NewBlocks) {
652  for (Instruction &I : *BB) {
653  RemapInstruction(&I, VMap,
655  }
656  }
657 
658  if (UseEpilogRemainder) {
659  // Connect the epilog code to the original loop and update the
660  // PHI functions.
661  ConnectEpilog(L, ModVal, NewExit, Exit, PreHeader,
662  EpilogPreHeader, NewPreHeader, VMap, DT, LI,
663  PreserveLCSSA);
664 
665  // Update counter in loop for unrolling.
666  // I should be multiply of Count.
667  IRBuilder<> B2(NewPreHeader->getTerminator());
668  Value *TestVal = B2.CreateSub(TripCount, ModVal, "unroll_iter");
669  BranchInst *LatchBR = cast<BranchInst>(Latch->getTerminator());
670  B2.SetInsertPoint(LatchBR);
671  PHINode *NewIdx = PHINode::Create(TestVal->getType(), 2, "niter",
672  Header->getFirstNonPHI());
673  Value *IdxSub =
674  B2.CreateSub(NewIdx, ConstantInt::get(NewIdx->getType(), 1),
675  NewIdx->getName() + ".nsub");
676  Value *IdxCmp;
677  if (LatchBR->getSuccessor(0) == Header)
678  IdxCmp = B2.CreateIsNotNull(IdxSub, NewIdx->getName() + ".ncmp");
679  else
680  IdxCmp = B2.CreateIsNull(IdxSub, NewIdx->getName() + ".ncmp");
681  NewIdx->addIncoming(TestVal, NewPreHeader);
682  NewIdx->addIncoming(IdxSub, Latch);
683  LatchBR->setCondition(IdxCmp);
684  } else {
685  // Connect the prolog code to the original loop and update the
686  // PHI functions.
687  ConnectProlog(L, BECount, Count, PrologExit, PreHeader, NewPreHeader,
688  VMap, DT, LI, PreserveLCSSA);
689  }
690 
691  // If this loop is nested, then the loop unroller changes the code in the
692  // parent loop, so the Scalar Evolution pass needs to be run again.
693  if (Loop *ParentLoop = L->getParentLoop())
694  SE->forgetLoop(ParentLoop);
695 
696  NumRuntimeUnrolled++;
697  return true;
698 }
MachineLoop * L
static unsigned getBitWidth(Type *Ty, const DataLayout &DL)
Returns the bitwidth of the given scalar or pointer type (if unknown returns 0).
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:76
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:102
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Definition: IRBuilder.h:699
BasicBlock * getUniqueExitBlock() const
If getUniqueExitBlocks would return exactly one block, return that block.
Definition: LoopInfo.cpp:399
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
LLVMContext & Context
Value * CreateIsNotNull(Value *Arg, const Twine &Name="")
Return an i1 value testing if Arg is not null.
Definition: IRBuilder.h:1704
const SCEV * getConstant(ConstantInt *V)
STATISTIC(NumFunctions,"Total number of functions")
size_t i
BasicBlock * SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr)
Split the specified block at the specified instruction - everything before SplitPt stays in Old and e...
Value * CreateICmpULT(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1478
void replaceOperandWith(unsigned I, Metadata *New)
Replace a specific operand.
Definition: Metadata.cpp:819
iterator end()
Definition: Function.h:537
ValueT lookup(const KeyT &Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: ValueMap.h:167
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:414
unsigned getNumOperands() const
Return number of MDNode operands.
Definition: Metadata.h:1040
The main scalar evolution driver.
This file contains the declarations for metadata subclasses.
bool isHighCostExpansion(const SCEV *Expr, Loop *L, const Instruction *At=nullptr)
Return true for expressions that may incur non-trivial cost to evaluate at runtime.
LoopT * getParentLoop() const
Definition: LoopInfo.h:103
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:100
Metadata node.
Definition: Metadata.h:830
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:575
BlockT * getHeader() const
Definition: LoopInfo.h:102
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:191
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
Definition: LoopInfoImpl.h:157
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:228
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
Remove an incoming value.
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr it the function does no...
Definition: BasicBlock.cpp:116
Instruction * getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:180
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:588
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:813
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:257
bool isLoopSimplifyForm() const
Return true if the Loop is in the form that the LoopSimplify form transforms loops to...
Definition: LoopInfo.cpp:190
#define F(x, y, z)
Definition: MD5.cpp:51
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool startswith(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:264
void addChildLoop(LoopT *NewChild)
Add the specified loop to be a child of this loop.
Definition: LoopInfo.h:279
BasicBlock * getSuccessor(unsigned i) const
void setSuccessor(unsigned idx, BasicBlock *B)
Update the specified successor to point at the provided block.
Definition: InstrTypes.h:84
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
void perform(LoopInfo *LI)
Traverse the loop blocks and store the DFS result.
Definition: LoopInfo.cpp:754
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:96
If this flag is set, the remapper knows that only local values within a function (such as an instruct...
Definition: ValueMapper.h:62
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:109
constexpr bool isPowerOf2_32(uint32_t Value)
isPowerOf2_32 - This function returns true if the argument is a power of two > 0. ...
Definition: MathExtras.h:399
LLVM Basic Block Representation.
Definition: BasicBlock.h:51
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:48
Type * getType() const
Return the LLVM type of this SCEV expression.
Conditional or Unconditional Branch instruction.
MDNode * getLoopID() const
Return the llvm.loop loop id metadata node for this loop if it is present.
Definition: LoopInfo.cpp:212
void splice(iterator where, iplist_impl &L2)
Definition: ilist.h:342
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
Definition: LoopInfo.h:109
Value * expandCodeFor(const SCEV *SH, Type *Ty, Instruction *I)
Insert code to directly compute the specified SCEV expression into the program.
std::vector< BasicBlock * >::const_reverse_iterator RPOIterator
Definition: LoopIterator.h:102
BlockT * getExitingBlock() const
If getExitingBlocks would return exactly one block, return that block.
Definition: LoopInfoImpl.h:52
self_iterator getIterator()
Definition: ilist_node.h:81
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1337
static void ConnectEpilog(Loop *L, Value *ModVal, BasicBlock *NewExit, BasicBlock *Exit, BasicBlock *PreHeader, BasicBlock *EpilogPreHeader, BasicBlock *NewPreHeader, ValueToValueMapTy &VMap, DominatorTree *DT, LoopInfo *LI, bool PreserveLCSSA)
Connect the unrolling epilog code to the original loop.
void setLoopID(MDNode *LoopID) const
Set the llvm.loop loop id metadata for this loop.
Definition: LoopInfo.cpp:246
StringRef getString() const
Definition: Metadata.cpp:424
const MDOperand & getOperand(unsigned I) const
Definition: Metadata.h:1034
Iterator for intrusive lists based on ilist_node.
const BasicBlockListType & getBasicBlockList() const
Definition: Function.h:512
void setIncomingBlock(unsigned i, BasicBlock *BB)
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:843
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:230
static void ConnectProlog(Loop *L, Value *BECount, unsigned Count, BasicBlock *PrologExit, BasicBlock *PreHeader, BasicBlock *NewPreHeader, ValueToValueMapTy &VMap, DominatorTree *DT, LoopInfo *LI, bool PreserveLCSSA)
Connect the unrolling prolog code to the original loop.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:558
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
pred_range predecessors(BasicBlock *BB)
Definition: IR/CFG.h:110
void addTopLevelLoop(LoopT *New)
This adds the specified loop to the collection of top-level loops.
Definition: LoopInfo.h:629
Value * CreateURem(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:911
unsigned Log2_32(uint32_t Value)
Log2_32 - This function returns the floor log base 2 of the specified value, -1 if the value is zero...
Definition: MathExtras.h:513
const Loop * addClonedBlockToLoopInfo(BasicBlock *OriginalBB, BasicBlock *ClonedBB, LoopInfo *LI, NewLoopsMap &NewLoops)
Adds ClonedBB to LoopInfo, creates a new loop for ClonedBB if necessary and adds a mapping from the o...
Definition: LoopUnroll.cpp:179
Store the result of a depth first search within basic blocks contained by a single loop...
Definition: LoopIterator.h:98
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:195
void RemapInstruction(Instruction *I, ValueToValueMapTy &VM, RemapFlags Flags=RF_None, ValueMapTypeRemapper *TypeMapper=nullptr, ValueMaterializer *Materializer=nullptr)
Convert the instruction operands from referencing the current values into those specified by VM...
Definition: ValueMapper.h:243
This class uses information about analyze scalars to rewrite expressions in canonical form...
const SCEV * getAddExpr(SmallVectorImpl< const SCEV * > &Ops, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap)
Get a canonical add expression, or something simpler if possible.
If this flag is set, the remapper ignores missing function-local entries (Argument, Instruction, BasicBlock) that are not in the value map.
Definition: ValueMapper.h:80
bool UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, bool AllowExpensiveTripCount, bool UseEpilogRemainder, LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT, bool PreserveLCSSA)
Insert code in the prolog/epilog code when unrolling a loop with a run-time trip-count.
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1132
use_iterator use_begin()
Definition: Value.h:310
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:384
void forgetLoop(const Loop *L)
This method should be called by the client when it has changed a loop in a way that may effect Scalar...
This class represents an analyzed expression in the program.
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:368
#define I(x, y, z)
Definition: MD5.cpp:54
TerminatorInst * getTerminator()
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:124
bool hasOneUse() const
Return true if there is exactly one user of this value.
Definition: Value.h:383
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:287
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:987
RPOIterator beginRPO() const
Reverse iterate over the cached postorder blocks.
Definition: LoopIterator.h:137
BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, bool PreserveLCSSA=false)
This method introduces at least one new basic block into the function and moves some of the predecess...
void setCondition(Value *V)
const SCEV * getBackedgeTakenCount(const Loop *L)
If the specified loop has a predictable backedge-taken count, return it, otherwise return a SCEVCould...
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LLVM Value Representation.
Definition: Value.h:71
succ_range successors(BasicBlock *BB)
Definition: IR/CFG.h:143
BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr)
Split the edge connecting specified block.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:47
static void CloneLoopBlocks(Loop *L, Value *NewIter, const bool CreateRemainderLoop, const bool UseEpilogRemainder, BasicBlock *InsertTop, BasicBlock *InsertBot, BasicBlock *Preheader, std::vector< BasicBlock * > &NewBlocks, LoopBlocksDFS &LoopBlocks, ValueToValueMapTy &VMap, LoopInfo *LI)
Create a clone of the blocks in a loop and connect them together.
A single uniqued string.
Definition: Metadata.h:586
void setIncomingValue(unsigned i, Value *V)
BasicBlock * CloneBasicBlock(const BasicBlock *BB, ValueToValueMapTy &VMap, const Twine &NameSuffix="", Function *F=nullptr, ClonedCodeInfo *CodeInfo=nullptr)
CloneBasicBlock - Return a copy of the specified basic block, but without embedding the block into a ...
int getBasicBlockIndex(const BasicBlock *BB) const
Return the first index of the specified basic block in the value list for this PHI.
RPOIterator endRPO() const
Definition: LoopIterator.h:141
bool erase(const KeyT &Val)
Definition: ValueMap.h:193