Bug Summary

File:lib/Transforms/Vectorize/VPlan.cpp
Warning:line 371, column 15
Value stored to 'VectorLatchBB' during its initialization is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name VPlan.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn345461/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/lib/Transforms/Vectorize -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-10-27-211344-32123-1 -x c++ /build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp -faddrsig
1//===- VPlan.cpp - Vectorizer Plan ----------------------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9///
10/// \file
11/// This is the LLVM vectorization plan. It represents a candidate for
12/// vectorization, allowing to plan and optimize how to vectorize a given loop
13/// before generating LLVM-IR.
14/// The vectorizer uses vectorization plans to estimate the costs of potential
15/// candidates and if profitable to execute the desired plan, generating vector
16/// LLVM-IR code.
17///
18//===----------------------------------------------------------------------===//
19
20#include "VPlan.h"
21#include "VPlanDominatorTree.h"
22#include "llvm/ADT/DepthFirstIterator.h"
23#include "llvm/ADT/PostOrderIterator.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/LoopInfo.h"
27#include "llvm/IR/BasicBlock.h"
28#include "llvm/IR/CFG.h"
29#include "llvm/IR/InstrTypes.h"
30#include "llvm/IR/Instruction.h"
31#include "llvm/IR/Instructions.h"
32#include "llvm/IR/Type.h"
33#include "llvm/IR/Value.h"
34#include "llvm/Support/Casting.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/ErrorHandling.h"
37#include "llvm/Support/GenericDomTreeConstruction.h"
38#include "llvm/Support/GraphWriter.h"
39#include "llvm/Support/raw_ostream.h"
40#include "llvm/Transforms/Utils/BasicBlockUtils.h"
41#include <cassert>
42#include <iterator>
43#include <string>
44#include <vector>
45
46using namespace llvm;
47extern cl::opt<bool> EnableVPlanNativePath;
48
49#define DEBUG_TYPE"vplan" "vplan"
50
51raw_ostream &llvm::operator<<(raw_ostream &OS, const VPValue &V) {
52 if (const VPInstruction *Instr = dyn_cast<VPInstruction>(&V))
53 Instr->print(OS);
54 else
55 V.printAsOperand(OS);
56 return OS;
57}
58
59/// \return the VPBasicBlock that is the entry of Block, possibly indirectly.
60const VPBasicBlock *VPBlockBase::getEntryBasicBlock() const {
61 const VPBlockBase *Block = this;
62 while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
63 Block = Region->getEntry();
64 return cast<VPBasicBlock>(Block);
65}
66
67VPBasicBlock *VPBlockBase::getEntryBasicBlock() {
68 VPBlockBase *Block = this;
69 while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
70 Block = Region->getEntry();
71 return cast<VPBasicBlock>(Block);
72}
73
74/// \return the VPBasicBlock that is the exit of Block, possibly indirectly.
75const VPBasicBlock *VPBlockBase::getExitBasicBlock() const {
76 const VPBlockBase *Block = this;
77 while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
78 Block = Region->getExit();
79 return cast<VPBasicBlock>(Block);
80}
81
82VPBasicBlock *VPBlockBase::getExitBasicBlock() {
83 VPBlockBase *Block = this;
84 while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
85 Block = Region->getExit();
86 return cast<VPBasicBlock>(Block);
87}
88
89VPBlockBase *VPBlockBase::getEnclosingBlockWithSuccessors() {
90 if (!Successors.empty() || !Parent)
91 return this;
92 assert(Parent->getExit() == this &&((Parent->getExit() == this && "Block w/o successors not the exit of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getExit() == this && \"Block w/o successors not the exit of its parent.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 93, __PRETTY_FUNCTION__))
93 "Block w/o successors not the exit of its parent.")((Parent->getExit() == this && "Block w/o successors not the exit of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getExit() == this && \"Block w/o successors not the exit of its parent.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 93, __PRETTY_FUNCTION__))
;
94 return Parent->getEnclosingBlockWithSuccessors();
95}
96
97VPBlockBase *VPBlockBase::getEnclosingBlockWithPredecessors() {
98 if (!Predecessors.empty() || !Parent)
99 return this;
100 assert(Parent->getEntry() == this &&((Parent->getEntry() == this && "Block w/o predecessors not the entry of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getEntry() == this && \"Block w/o predecessors not the entry of its parent.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 101, __PRETTY_FUNCTION__))
101 "Block w/o predecessors not the entry of its parent.")((Parent->getEntry() == this && "Block w/o predecessors not the entry of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getEntry() == this && \"Block w/o predecessors not the entry of its parent.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 101, __PRETTY_FUNCTION__))
;
102 return Parent->getEnclosingBlockWithPredecessors();
103}
104
105void VPBlockBase::deleteCFG(VPBlockBase *Entry) {
106 SmallVector<VPBlockBase *, 8> Blocks;
107 for (VPBlockBase *Block : depth_first(Entry))
108 Blocks.push_back(Block);
109
110 for (VPBlockBase *Block : Blocks)
111 delete Block;
112}
113
114BasicBlock *
115VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
116 // BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks.
117 // Pred stands for Predessor. Prev stands for Previous - last visited/created.
118 BasicBlock *PrevBB = CFG.PrevBB;
119 BasicBlock *NewBB = BasicBlock::Create(PrevBB->getContext(), getName(),
120 PrevBB->getParent(), CFG.LastBB);
121 LLVM_DEBUG(dbgs() << "LV: created " << NewBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: created " << NewBB->
getName() << '\n'; } } while (false)
;
122
123 // Hook up the new basic block to its predecessors.
124 for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
125 VPBasicBlock *PredVPBB = PredVPBlock->getExitBasicBlock();
126 auto &PredVPSuccessors = PredVPBB->getSuccessors();
127 BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
128
129 // In outer loop vectorization scenario, the predecessor BBlock may not yet
130 // be visited(backedge). Mark the VPBasicBlock for fixup at the end of
131 // vectorization. We do not encounter this case in inner loop vectorization
132 // as we start out by building a loop skeleton with the vector loop header
133 // and latch blocks. As a result, we never enter this function for the
134 // header block in the non VPlan-native path.
135 if (!PredBB) {
136 assert(EnableVPlanNativePath &&((EnableVPlanNativePath && "Unexpected null predecessor in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected null predecessor in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 137, __PRETTY_FUNCTION__))
137 "Unexpected null predecessor in non VPlan-native path")((EnableVPlanNativePath && "Unexpected null predecessor in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected null predecessor in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 137, __PRETTY_FUNCTION__))
;
138 CFG.VPBBsToFix.push_back(PredVPBB);
139 continue;
140 }
141
142 assert(PredBB && "Predecessor basic-block not found building successor.")((PredBB && "Predecessor basic-block not found building successor."
) ? static_cast<void> (0) : __assert_fail ("PredBB && \"Predecessor basic-block not found building successor.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 142, __PRETTY_FUNCTION__))
;
143 auto *PredBBTerminator = PredBB->getTerminator();
144 LLVM_DEBUG(dbgs() << "LV: draw edge from" << PredBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: draw edge from" << PredBB
->getName() << '\n'; } } while (false)
;
145 if (isa<UnreachableInst>(PredBBTerminator)) {
146 assert(PredVPSuccessors.size() == 1 &&((PredVPSuccessors.size() == 1 && "Predecessor ending w/o branch must have single successor."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 1 && \"Predecessor ending w/o branch must have single successor.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 147, __PRETTY_FUNCTION__))
147 "Predecessor ending w/o branch must have single successor.")((PredVPSuccessors.size() == 1 && "Predecessor ending w/o branch must have single successor."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 1 && \"Predecessor ending w/o branch must have single successor.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 147, __PRETTY_FUNCTION__))
;
148 PredBBTerminator->eraseFromParent();
149 BranchInst::Create(NewBB, PredBB);
150 } else {
151 assert(PredVPSuccessors.size() == 2 &&((PredVPSuccessors.size() == 2 && "Predecessor ending with branch must have two successors."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 2 && \"Predecessor ending with branch must have two successors.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 152, __PRETTY_FUNCTION__))
152 "Predecessor ending with branch must have two successors.")((PredVPSuccessors.size() == 2 && "Predecessor ending with branch must have two successors."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 2 && \"Predecessor ending with branch must have two successors.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 152, __PRETTY_FUNCTION__))
;
153 unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
154 assert(!PredBBTerminator->getSuccessor(idx) &&((!PredBBTerminator->getSuccessor(idx) && "Trying to reset an existing successor block."
) ? static_cast<void> (0) : __assert_fail ("!PredBBTerminator->getSuccessor(idx) && \"Trying to reset an existing successor block.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 155, __PRETTY_FUNCTION__))
155 "Trying to reset an existing successor block.")((!PredBBTerminator->getSuccessor(idx) && "Trying to reset an existing successor block."
) ? static_cast<void> (0) : __assert_fail ("!PredBBTerminator->getSuccessor(idx) && \"Trying to reset an existing successor block.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 155, __PRETTY_FUNCTION__))
;
156 PredBBTerminator->setSuccessor(idx, NewBB);
157 }
158 }
159 return NewBB;
160}
161
162void VPBasicBlock::execute(VPTransformState *State) {
163 bool Replica = State->Instance &&
164 !(State->Instance->Part == 0 && State->Instance->Lane == 0);
165 VPBasicBlock *PrevVPBB = State->CFG.PrevVPBB;
166 VPBlockBase *SingleHPred = nullptr;
167 BasicBlock *NewBB = State->CFG.PrevBB; // Reuse it if possible.
168
169 // 1. Create an IR basic block, or reuse the last one if possible.
170 // The last IR basic block is reused, as an optimization, in three cases:
171 // A. the first VPBB reuses the loop header BB - when PrevVPBB is null;
172 // B. when the current VPBB has a single (hierarchical) predecessor which
173 // is PrevVPBB and the latter has a single (hierarchical) successor; and
174 // C. when the current VPBB is an entry of a region replica - where PrevVPBB
175 // is the exit of this region from a previous instance, or the predecessor
176 // of this region.
177 if (PrevVPBB && /* A */
178 !((SingleHPred = getSingleHierarchicalPredecessor()) &&
179 SingleHPred->getExitBasicBlock() == PrevVPBB &&
180 PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */
181 !(Replica && getPredecessors().empty())) { /* C */
182 NewBB = createEmptyBasicBlock(State->CFG);
183 State->Builder.SetInsertPoint(NewBB);
184 // Temporarily terminate with unreachable until CFG is rewired.
185 UnreachableInst *Terminator = State->Builder.CreateUnreachable();
186 State->Builder.SetInsertPoint(Terminator);
187 // Register NewBB in its loop. In innermost loops its the same for all BB's.
188 Loop *L = State->LI->getLoopFor(State->CFG.LastBB);
189 L->addBasicBlockToLoop(NewBB, *State->LI);
190 State->CFG.PrevBB = NewBB;
191 }
192
193 // 2. Fill the IR basic block with IR instructions.
194 LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: vectorizing VPBB:" <<
getName() << " in BB:" << NewBB->getName() <<
'\n'; } } while (false)
195 << " in BB:" << NewBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: vectorizing VPBB:" <<
getName() << " in BB:" << NewBB->getName() <<
'\n'; } } while (false)
;
196
197 State->CFG.VPBB2IRBB[this] = NewBB;
198 State->CFG.PrevVPBB = this;
199
200 for (VPRecipeBase &Recipe : Recipes)
201 Recipe.execute(*State);
202
203 VPValue *CBV;
204 if (EnableVPlanNativePath && (CBV = getCondBit())) {
205 Value *IRCBV = CBV->getUnderlyingValue();
206 assert(IRCBV && "Unexpected null underlying value for condition bit")((IRCBV && "Unexpected null underlying value for condition bit"
) ? static_cast<void> (0) : __assert_fail ("IRCBV && \"Unexpected null underlying value for condition bit\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 206, __PRETTY_FUNCTION__))
;
207
208 // Condition bit value in a VPBasicBlock is used as the branch selector. In
209 // the VPlan-native path case, since all branches are uniform we generate a
210 // branch instruction using the condition value from vector lane 0 and dummy
211 // successors. The successors are fixed later when the successor blocks are
212 // visited.
213 Value *NewCond = State->Callback.getOrCreateVectorValues(IRCBV, 0);
214 NewCond = State->Builder.CreateExtractElement(NewCond,
215 State->Builder.getInt32(0));
216
217 // Replace the temporary unreachable terminator with the new conditional
218 // branch.
219 auto *CurrentTerminator = NewBB->getTerminator();
220 assert(isa<UnreachableInst>(CurrentTerminator) &&((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional "
"branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 222, __PRETTY_FUNCTION__))
221 "Expected to replace unreachable terminator with conditional "((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional "
"branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 222, __PRETTY_FUNCTION__))
222 "branch.")((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional "
"branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 222, __PRETTY_FUNCTION__))
;
223 auto *CondBr = BranchInst::Create(NewBB, nullptr, NewCond);
224 CondBr->setSuccessor(0, nullptr);
225 ReplaceInstWithInst(CurrentTerminator, CondBr);
226 }
227
228 LLVM_DEBUG(dbgs() << "LV: filled BB:" << *NewBB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: filled BB:" << *NewBB
; } } while (false)
;
229}
230
231void VPRegionBlock::execute(VPTransformState *State) {
232 ReversePostOrderTraversal<VPBlockBase *> RPOT(Entry);
233
234 if (!isReplicator()) {
235 // Visit the VPBlocks connected to "this", starting from it.
236 for (VPBlockBase *Block : RPOT) {
237 if (EnableVPlanNativePath) {
238 // The inner loop vectorization path does not represent loop preheader
239 // and exit blocks as part of the VPlan. In the VPlan-native path, skip
240 // vectorizing loop preheader block. In future, we may replace this
241 // check with the check for loop preheader.
242 if (Block->getNumPredecessors() == 0)
243 continue;
244
245 // Skip vectorizing loop exit block. In future, we may replace this
246 // check with the check for loop exit.
247 if (Block->getNumSuccessors() == 0)
248 continue;
249 }
250
251 LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: VPBlock in RPO " << Block
->getName() << '\n'; } } while (false)
;
252 Block->execute(State);
253 }
254 return;
255 }
256
257 assert(!State->Instance && "Replicating a Region with non-null instance.")((!State->Instance && "Replicating a Region with non-null instance."
) ? static_cast<void> (0) : __assert_fail ("!State->Instance && \"Replicating a Region with non-null instance.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 257, __PRETTY_FUNCTION__))
;
258
259 // Enter replicating mode.
260 State->Instance = {0, 0};
261
262 for (unsigned Part = 0, UF = State->UF; Part < UF; ++Part) {
263 State->Instance->Part = Part;
264 for (unsigned Lane = 0, VF = State->VF; Lane < VF; ++Lane) {
265 State->Instance->Lane = Lane;
266 // Visit the VPBlocks connected to \p this, starting from it.
267 for (VPBlockBase *Block : RPOT) {
268 LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: VPBlock in RPO " << Block
->getName() << '\n'; } } while (false)
;
269 Block->execute(State);
270 }
271 }
272 }
273
274 // Exit replicating mode.
275 State->Instance.reset();
276}
277
278void VPRecipeBase::insertBefore(VPRecipeBase *InsertPos) {
279 Parent = InsertPos->getParent();
280 Parent->getRecipeList().insert(InsertPos->getIterator(), this);
281}
282
283iplist<VPRecipeBase>::iterator VPRecipeBase::eraseFromParent() {
284 return getParent()->getRecipeList().erase(getIterator());
285}
286
287void VPInstruction::generateInstruction(VPTransformState &State,
288 unsigned Part) {
289 IRBuilder<> &Builder = State.Builder;
290
291 if (Instruction::isBinaryOp(getOpcode())) {
292 Value *A = State.get(getOperand(0), Part);
293 Value *B = State.get(getOperand(1), Part);
294 Value *V = Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B);
295 State.set(this, V, Part);
296 return;
297 }
298
299 switch (getOpcode()) {
300 case VPInstruction::Not: {
301 Value *A = State.get(getOperand(0), Part);
302 Value *V = Builder.CreateNot(A);
303 State.set(this, V, Part);
304 break;
305 }
306 case VPInstruction::ICmpULE: {
307 Value *IV = State.get(getOperand(0), Part);
308 Value *TC = State.get(getOperand(1), Part);
309 Value *V = Builder.CreateICmpULE(IV, TC);
310 State.set(this, V, Part);
311 break;
312 }
313 default:
314 llvm_unreachable("Unsupported opcode for instruction")::llvm::llvm_unreachable_internal("Unsupported opcode for instruction"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 314)
;
315 }
316}
317
318void VPInstruction::execute(VPTransformState &State) {
319 assert(!State.Instance && "VPInstruction executing an Instance")((!State.Instance && "VPInstruction executing an Instance"
) ? static_cast<void> (0) : __assert_fail ("!State.Instance && \"VPInstruction executing an Instance\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 319, __PRETTY_FUNCTION__))
;
320 for (unsigned Part = 0; Part < State.UF; ++Part)
321 generateInstruction(State, Part);
322}
323
324void VPInstruction::print(raw_ostream &O, const Twine &Indent) const {
325 O << " +\n" << Indent << "\"EMIT ";
326 print(O);
327 O << "\\l\"";
328}
329
330void VPInstruction::print(raw_ostream &O) const {
331 printAsOperand(O);
332 O << " = ";
333
334 switch (getOpcode()) {
335 case VPInstruction::Not:
336 O << "not";
337 break;
338 case VPInstruction::ICmpULE:
339 O << "icmp ule";
340 break;
341 default:
342 O << Instruction::getOpcodeName(getOpcode());
343 }
344
345 for (const VPValue *Operand : operands()) {
346 O << " ";
347 Operand->printAsOperand(O);
348 }
349}
350
351/// Generate the code inside the body of the vectorized loop. Assumes a single
352/// LoopVectorBody basic-block was created for this. Introduce additional
353/// basic-blocks as needed, and fill them all.
354void VPlan::execute(VPTransformState *State) {
355 // -1. Check if the backedge taken count is needed, and if so build it.
356 if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
357 Value *TC = State->TripCount;
358 IRBuilder<> Builder(State->CFG.PrevBB->getTerminator());
359 auto *TCMO = Builder.CreateSub(TC, ConstantInt::get(TC->getType(), 1),
360 "trip.count.minus.1");
361 Value2VPValue[TCMO] = BackedgeTakenCount;
362 }
363
364 // 0. Set the reverse mapping from VPValues to Values for code generation.
365 for (auto &Entry : Value2VPValue)
366 State->VPValue2Value[Entry.second] = Entry.first;
367
368 BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB;
369 BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor();
370 assert(VectorHeaderBB && "Loop preheader does not have a single successor.")((VectorHeaderBB && "Loop preheader does not have a single successor."
) ? static_cast<void> (0) : __assert_fail ("VectorHeaderBB && \"Loop preheader does not have a single successor.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 370, __PRETTY_FUNCTION__))
;
371 BasicBlock *VectorLatchBB = VectorHeaderBB;
Value stored to 'VectorLatchBB' during its initialization is never read
372
373 // 1. Make room to generate basic-blocks inside loop body if needed.
374 VectorLatchBB = VectorHeaderBB->splitBasicBlock(
375 VectorHeaderBB->getFirstInsertionPt(), "vector.body.latch");
376 Loop *L = State->LI->getLoopFor(VectorHeaderBB);
377 L->addBasicBlockToLoop(VectorLatchBB, *State->LI);
378 // Remove the edge between Header and Latch to allow other connections.
379 // Temporarily terminate with unreachable until CFG is rewired.
380 // Note: this asserts the generated code's assumption that
381 // getFirstInsertionPt() can be dereferenced into an Instruction.
382 VectorHeaderBB->getTerminator()->eraseFromParent();
383 State->Builder.SetInsertPoint(VectorHeaderBB);
384 UnreachableInst *Terminator = State->Builder.CreateUnreachable();
385 State->Builder.SetInsertPoint(Terminator);
386
387 // 2. Generate code in loop body.
388 State->CFG.PrevVPBB = nullptr;
389 State->CFG.PrevBB = VectorHeaderBB;
390 State->CFG.LastBB = VectorLatchBB;
391
392 for (VPBlockBase *Block : depth_first(Entry))
393 Block->execute(State);
394
395 // Setup branch terminator successors for VPBBs in VPBBsToFix based on
396 // VPBB's successors.
397 for (auto VPBB : State->CFG.VPBBsToFix) {
398 assert(EnableVPlanNativePath &&((EnableVPlanNativePath && "Unexpected VPBBsToFix in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected VPBBsToFix in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 399, __PRETTY_FUNCTION__))
399 "Unexpected VPBBsToFix in non VPlan-native path")((EnableVPlanNativePath && "Unexpected VPBBsToFix in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected VPBBsToFix in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 399, __PRETTY_FUNCTION__))
;
400 BasicBlock *BB = State->CFG.VPBB2IRBB[VPBB];
401 assert(BB && "Unexpected null basic block for VPBB")((BB && "Unexpected null basic block for VPBB") ? static_cast
<void> (0) : __assert_fail ("BB && \"Unexpected null basic block for VPBB\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 401, __PRETTY_FUNCTION__))
;
402
403 unsigned Idx = 0;
404 auto *BBTerminator = BB->getTerminator();
405
406 for (VPBlockBase *SuccVPBlock : VPBB->getHierarchicalSuccessors()) {
407 VPBasicBlock *SuccVPBB = SuccVPBlock->getEntryBasicBlock();
408 BBTerminator->setSuccessor(Idx, State->CFG.VPBB2IRBB[SuccVPBB]);
409 ++Idx;
410 }
411 }
412
413 // 3. Merge the temporary latch created with the last basic-block filled.
414 BasicBlock *LastBB = State->CFG.PrevBB;
415 // Connect LastBB to VectorLatchBB to facilitate their merge.
416 assert((EnableVPlanNativePath ||(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB
->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable"
) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 418, __PRETTY_FUNCTION__))
417 isa<UnreachableInst>(LastBB->getTerminator())) &&(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB
->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable"
) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 418, __PRETTY_FUNCTION__))
418 "Expected InnerLoop VPlan CFG to terminate with unreachable")(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB
->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable"
) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 418, __PRETTY_FUNCTION__))
;
419 assert((!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) &&(((!EnableVPlanNativePath || isa<BranchInst>(LastBB->
getTerminator())) && "Expected VPlan CFG to terminate with branch in NativePath"
) ? static_cast<void> (0) : __assert_fail ("(!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) && \"Expected VPlan CFG to terminate with branch in NativePath\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 420, __PRETTY_FUNCTION__))
420 "Expected VPlan CFG to terminate with branch in NativePath")(((!EnableVPlanNativePath || isa<BranchInst>(LastBB->
getTerminator())) && "Expected VPlan CFG to terminate with branch in NativePath"
) ? static_cast<void> (0) : __assert_fail ("(!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) && \"Expected VPlan CFG to terminate with branch in NativePath\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 420, __PRETTY_FUNCTION__))
;
421 LastBB->getTerminator()->eraseFromParent();
422 BranchInst::Create(VectorLatchBB, LastBB);
423
424 // Merge LastBB with Latch.
425 bool Merged = MergeBlockIntoPredecessor(VectorLatchBB, nullptr, State->LI);
426 (void)Merged;
427 assert(Merged && "Could not merge last basic block with latch.")((Merged && "Could not merge last basic block with latch."
) ? static_cast<void> (0) : __assert_fail ("Merged && \"Could not merge last basic block with latch.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 427, __PRETTY_FUNCTION__))
;
428 VectorLatchBB = LastBB;
429
430 // We do not attempt to preserve DT for outer loop vectorization currently.
431 if (!EnableVPlanNativePath)
432 updateDominatorTree(State->DT, VectorPreHeaderBB, VectorLatchBB);
433}
434
435void VPlan::updateDominatorTree(DominatorTree *DT, BasicBlock *LoopPreHeaderBB,
436 BasicBlock *LoopLatchBB) {
437 BasicBlock *LoopHeaderBB = LoopPreHeaderBB->getSingleSuccessor();
438 assert(LoopHeaderBB && "Loop preheader does not have a single successor.")((LoopHeaderBB && "Loop preheader does not have a single successor."
) ? static_cast<void> (0) : __assert_fail ("LoopHeaderBB && \"Loop preheader does not have a single successor.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 438, __PRETTY_FUNCTION__))
;
439 DT->addNewBlock(LoopHeaderBB, LoopPreHeaderBB);
440 // The vector body may be more than a single basic-block by this point.
441 // Update the dominator tree information inside the vector body by propagating
442 // it from header to latch, expecting only triangular control-flow, if any.
443 BasicBlock *PostDomSucc = nullptr;
444 for (auto *BB = LoopHeaderBB; BB != LoopLatchBB; BB = PostDomSucc) {
445 // Get the list of successors of this block.
446 std::vector<BasicBlock *> Succs(succ_begin(BB), succ_end(BB));
447 assert(Succs.size() <= 2 &&((Succs.size() <= 2 && "Basic block in vector loop has more than 2 successors."
) ? static_cast<void> (0) : __assert_fail ("Succs.size() <= 2 && \"Basic block in vector loop has more than 2 successors.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 448, __PRETTY_FUNCTION__))
448 "Basic block in vector loop has more than 2 successors.")((Succs.size() <= 2 && "Basic block in vector loop has more than 2 successors."
) ? static_cast<void> (0) : __assert_fail ("Succs.size() <= 2 && \"Basic block in vector loop has more than 2 successors.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 448, __PRETTY_FUNCTION__))
;
449 PostDomSucc = Succs[0];
450 if (Succs.size() == 1) {
451 assert(PostDomSucc->getSinglePredecessor() &&((PostDomSucc->getSinglePredecessor() && "PostDom successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->getSinglePredecessor() && \"PostDom successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 452, __PRETTY_FUNCTION__))
452 "PostDom successor has more than one predecessor.")((PostDomSucc->getSinglePredecessor() && "PostDom successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->getSinglePredecessor() && \"PostDom successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 452, __PRETTY_FUNCTION__))
;
453 DT->addNewBlock(PostDomSucc, BB);
454 continue;
455 }
456 BasicBlock *InterimSucc = Succs[1];
457 if (PostDomSucc->getSingleSuccessor() == InterimSucc) {
458 PostDomSucc = Succs[1];
459 InterimSucc = Succs[0];
460 }
461 assert(InterimSucc->getSingleSuccessor() == PostDomSucc &&((InterimSucc->getSingleSuccessor() == PostDomSucc &&
"One successor of a basic block does not lead to the other."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSingleSuccessor() == PostDomSucc && \"One successor of a basic block does not lead to the other.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 462, __PRETTY_FUNCTION__))
462 "One successor of a basic block does not lead to the other.")((InterimSucc->getSingleSuccessor() == PostDomSucc &&
"One successor of a basic block does not lead to the other."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSingleSuccessor() == PostDomSucc && \"One successor of a basic block does not lead to the other.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 462, __PRETTY_FUNCTION__))
;
463 assert(InterimSucc->getSinglePredecessor() &&((InterimSucc->getSinglePredecessor() && "Interim successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSinglePredecessor() && \"Interim successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 464, __PRETTY_FUNCTION__))
464 "Interim successor has more than one predecessor.")((InterimSucc->getSinglePredecessor() && "Interim successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSinglePredecessor() && \"Interim successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 464, __PRETTY_FUNCTION__))
;
465 assert(pred_size(PostDomSucc) == 2 &&((pred_size(PostDomSucc) == 2 && "PostDom successor has more than two predecessors."
) ? static_cast<void> (0) : __assert_fail ("pred_size(PostDomSucc) == 2 && \"PostDom successor has more than two predecessors.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 466, __PRETTY_FUNCTION__))
466 "PostDom successor has more than two predecessors.")((pred_size(PostDomSucc) == 2 && "PostDom successor has more than two predecessors."
) ? static_cast<void> (0) : __assert_fail ("pred_size(PostDomSucc) == 2 && \"PostDom successor has more than two predecessors.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 466, __PRETTY_FUNCTION__))
;
467 DT->addNewBlock(InterimSucc, BB);
468 DT->addNewBlock(PostDomSucc, BB);
469 }
470}
471
472const Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
473 return (isa<VPRegionBlock>(Block) ? "cluster_N" : "N") +
474 Twine(getOrCreateBID(Block));
475}
476
477const Twine VPlanPrinter::getOrCreateName(const VPBlockBase *Block) {
478 const std::string &Name = Block->getName();
479 if (!Name.empty())
480 return Name;
481 return "VPB" + Twine(getOrCreateBID(Block));
482}
483
484void VPlanPrinter::dump() {
485 Depth = 1;
486 bumpIndent(0);
487 OS << "digraph VPlan {\n";
488 OS << "graph [labelloc=t, fontsize=30; label=\"Vectorization Plan";
489 if (!Plan.getName().empty())
490 OS << "\\n" << DOT::EscapeString(Plan.getName());
491 if (!Plan.Value2VPValue.empty() || Plan.BackedgeTakenCount) {
492 OS << ", where:";
493 if (Plan.BackedgeTakenCount)
494 OS << "\\n"
495 << *Plan.getOrCreateBackedgeTakenCount() << " := BackedgeTakenCount";
496 for (auto Entry : Plan.Value2VPValue) {
497 OS << "\\n" << *Entry.second;
498 OS << DOT::EscapeString(" := ");
499 Entry.first->printAsOperand(OS, false);
500 }
501 }
502 OS << "\"]\n";
503 OS << "node [shape=rect, fontname=Courier, fontsize=30]\n";
504 OS << "edge [fontname=Courier, fontsize=30]\n";
505 OS << "compound=true\n";
506
507 for (VPBlockBase *Block : depth_first(Plan.getEntry()))
508 dumpBlock(Block);
509
510 OS << "}\n";
511}
512
513void VPlanPrinter::dumpBlock(const VPBlockBase *Block) {
514 if (const VPBasicBlock *BasicBlock = dyn_cast<VPBasicBlock>(Block))
515 dumpBasicBlock(BasicBlock);
516 else if (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
517 dumpRegion(Region);
518 else
519 llvm_unreachable("Unsupported kind of VPBlock.")::llvm::llvm_unreachable_internal("Unsupported kind of VPBlock."
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 519)
;
520}
521
522void VPlanPrinter::drawEdge(const VPBlockBase *From, const VPBlockBase *To,
523 bool Hidden, const Twine &Label) {
524 // Due to "dot" we print an edge between two regions as an edge between the
525 // exit basic block and the entry basic of the respective regions.
526 const VPBlockBase *Tail = From->getExitBasicBlock();
527 const VPBlockBase *Head = To->getEntryBasicBlock();
528 OS << Indent << getUID(Tail) << " -> " << getUID(Head);
529 OS << " [ label=\"" << Label << '\"';
530 if (Tail != From)
531 OS << " ltail=" << getUID(From);
532 if (Head != To)
533 OS << " lhead=" << getUID(To);
534 if (Hidden)
535 OS << "; splines=none";
536 OS << "]\n";
537}
538
539void VPlanPrinter::dumpEdges(const VPBlockBase *Block) {
540 auto &Successors = Block->getSuccessors();
541 if (Successors.size() == 1)
542 drawEdge(Block, Successors.front(), false, "");
543 else if (Successors.size() == 2) {
544 drawEdge(Block, Successors.front(), false, "T");
545 drawEdge(Block, Successors.back(), false, "F");
546 } else {
547 unsigned SuccessorNumber = 0;
548 for (auto *Successor : Successors)
549 drawEdge(Block, Successor, false, Twine(SuccessorNumber++));
550 }
551}
552
553void VPlanPrinter::dumpBasicBlock(const VPBasicBlock *BasicBlock) {
554 OS << Indent << getUID(BasicBlock) << " [label =\n";
555 bumpIndent(1);
556 OS << Indent << "\"" << DOT::EscapeString(BasicBlock->getName()) << ":\\n\"";
557 bumpIndent(1);
558 for (const VPRecipeBase &Recipe : *BasicBlock)
559 Recipe.print(OS, Indent);
560
561 // Dump the condition bit.
562 const VPValue *CBV = BasicBlock->getCondBit();
563 if (CBV) {
564 OS << " +\n" << Indent << " \"CondBit: ";
565 if (const VPInstruction *CBI = dyn_cast<VPInstruction>(CBV)) {
566 CBI->printAsOperand(OS);
567 OS << " (" << DOT::EscapeString(CBI->getParent()->getName()) << ")\\l\"";
568 } else {
569 CBV->printAsOperand(OS);
570 OS << "\"";
571 }
572 }
573
574 bumpIndent(-2);
575 OS << "\n" << Indent << "]\n";
576 dumpEdges(BasicBlock);
577}
578
579void VPlanPrinter::dumpRegion(const VPRegionBlock *Region) {
580 OS << Indent << "subgraph " << getUID(Region) << " {\n";
581 bumpIndent(1);
582 OS << Indent << "fontname=Courier\n"
583 << Indent << "label=\""
584 << DOT::EscapeString(Region->isReplicator() ? "<xVFxUF> " : "<x1> ")
585 << DOT::EscapeString(Region->getName()) << "\"\n";
586 // Dump the blocks of the region.
587 assert(Region->getEntry() && "Region contains no inner blocks.")((Region->getEntry() && "Region contains no inner blocks."
) ? static_cast<void> (0) : __assert_fail ("Region->getEntry() && \"Region contains no inner blocks.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Vectorize/VPlan.cpp"
, 587, __PRETTY_FUNCTION__))
;
588 for (const VPBlockBase *Block : depth_first(Region->getEntry()))
589 dumpBlock(Block);
590 bumpIndent(-1);
591 OS << Indent << "}\n";
592 dumpEdges(Region);
593}
594
595void VPlanPrinter::printAsIngredient(raw_ostream &O, Value *V) {
596 std::string IngredientString;
597 raw_string_ostream RSO(IngredientString);
598 if (auto *Inst = dyn_cast<Instruction>(V)) {
599 if (!Inst->getType()->isVoidTy()) {
600 Inst->printAsOperand(RSO, false);
601 RSO << " = ";
602 }
603 RSO << Inst->getOpcodeName() << " ";
604 unsigned E = Inst->getNumOperands();
605 if (E > 0) {
606 Inst->getOperand(0)->printAsOperand(RSO, false);
607 for (unsigned I = 1; I < E; ++I)
608 Inst->getOperand(I)->printAsOperand(RSO << ", ", false);
609 }
610 } else // !Inst
611 V->printAsOperand(RSO, false);
612 RSO.flush();
613 O << DOT::EscapeString(IngredientString);
614}
615
616void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent) const {
617 O << " +\n" << Indent << "\"WIDEN\\l\"";
618 for (auto &Instr : make_range(Begin, End))
619 O << " +\n" << Indent << "\" " << VPlanIngredient(&Instr) << "\\l\"";
620}
621
622void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O,
623 const Twine &Indent) const {
624 O << " +\n" << Indent << "\"WIDEN-INDUCTION";
625 if (Trunc) {
626 O << "\\l\"";
627 O << " +\n" << Indent << "\" " << VPlanIngredient(IV) << "\\l\"";
628 O << " +\n" << Indent << "\" " << VPlanIngredient(Trunc) << "\\l\"";
629 } else
630 O << " " << VPlanIngredient(IV) << "\\l\"";
631}
632
633void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent) const {
634 O << " +\n" << Indent << "\"WIDEN-PHI " << VPlanIngredient(Phi) << "\\l\"";
635}
636
637void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent) const {
638 O << " +\n" << Indent << "\"BLEND ";
639 Phi->printAsOperand(O, false);
640 O << " =";
641 if (!User) {
642 // Not a User of any mask: not really blending, this is a
643 // single-predecessor phi.
644 O << " ";
645 Phi->getIncomingValue(0)->printAsOperand(O, false);
646 } else {
647 for (unsigned I = 0, E = User->getNumOperands(); I < E; ++I) {
648 O << " ";
649 Phi->getIncomingValue(I)->printAsOperand(O, false);
650 O << "/";
651 User->getOperand(I)->printAsOperand(O);
652 }
653 }
654 O << "\\l\"";
655}
656
657void VPReplicateRecipe::print(raw_ostream &O, const Twine &Indent) const {
658 O << " +\n"
659 << Indent << "\"" << (IsUniform ? "CLONE " : "REPLICATE ")
660 << VPlanIngredient(Ingredient);
661 if (AlsoPack)
662 O << " (S->V)";
663 O << "\\l\"";
664}
665
666void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent) const {
667 O << " +\n"
668 << Indent << "\"PHI-PREDICATED-INSTRUCTION " << VPlanIngredient(PredInst)
669 << "\\l\"";
670}
671
672void VPWidenMemoryInstructionRecipe::print(raw_ostream &O,
673 const Twine &Indent) const {
674 O << " +\n" << Indent << "\"WIDEN " << VPlanIngredient(&Instr);
675 if (User) {
676 O << ", ";
677 User->getOperand(0)->printAsOperand(O);
678 }
679 O << "\\l\"";
680}
681
682template void DomTreeBuilder::Calculate<VPDominatorTree>(VPDominatorTree &DT);