Bug Summary

File:lib/Transforms/Vectorize/VPlan.cpp
Warning:line 377, column 15
Value stored to 'VectorLatchBB' during its initialization is never read

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name VPlan.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize -I /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn350071/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/lib/Transforms/Vectorize -fdebug-prefix-map=/build/llvm-toolchain-snapshot-8~svn350071=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-12-27-042839-1215-1 -x c++ /build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp -faddrsig
1//===- VPlan.cpp - Vectorizer Plan ----------------------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9///
10/// \file
11/// This is the LLVM vectorization plan. It represents a candidate for
12/// vectorization, allowing to plan and optimize how to vectorize a given loop
13/// before generating LLVM-IR.
14/// The vectorizer uses vectorization plans to estimate the costs of potential
15/// candidates and if profitable to execute the desired plan, generating vector
16/// LLVM-IR code.
17///
18//===----------------------------------------------------------------------===//
19
20#include "VPlan.h"
21#include "VPlanDominatorTree.h"
22#include "llvm/ADT/DepthFirstIterator.h"
23#include "llvm/ADT/PostOrderIterator.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/Twine.h"
26#include "llvm/Analysis/LoopInfo.h"
27#include "llvm/IR/BasicBlock.h"
28#include "llvm/IR/CFG.h"
29#include "llvm/IR/InstrTypes.h"
30#include "llvm/IR/Instruction.h"
31#include "llvm/IR/Instructions.h"
32#include "llvm/IR/Type.h"
33#include "llvm/IR/Value.h"
34#include "llvm/Support/Casting.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/ErrorHandling.h"
37#include "llvm/Support/GenericDomTreeConstruction.h"
38#include "llvm/Support/GraphWriter.h"
39#include "llvm/Support/raw_ostream.h"
40#include "llvm/Transforms/Utils/BasicBlockUtils.h"
41#include <cassert>
42#include <iterator>
43#include <string>
44#include <vector>
45
46using namespace llvm;
47extern cl::opt<bool> EnableVPlanNativePath;
48
49#define DEBUG_TYPE"vplan" "vplan"
50
51raw_ostream &llvm::operator<<(raw_ostream &OS, const VPValue &V) {
52 if (const VPInstruction *Instr = dyn_cast<VPInstruction>(&V))
53 Instr->print(OS);
54 else
55 V.printAsOperand(OS);
56 return OS;
57}
58
59/// \return the VPBasicBlock that is the entry of Block, possibly indirectly.
60const VPBasicBlock *VPBlockBase::getEntryBasicBlock() const {
61 const VPBlockBase *Block = this;
62 while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
63 Block = Region->getEntry();
64 return cast<VPBasicBlock>(Block);
65}
66
67VPBasicBlock *VPBlockBase::getEntryBasicBlock() {
68 VPBlockBase *Block = this;
69 while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
70 Block = Region->getEntry();
71 return cast<VPBasicBlock>(Block);
72}
73
74/// \return the VPBasicBlock that is the exit of Block, possibly indirectly.
75const VPBasicBlock *VPBlockBase::getExitBasicBlock() const {
76 const VPBlockBase *Block = this;
77 while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
78 Block = Region->getExit();
79 return cast<VPBasicBlock>(Block);
80}
81
82VPBasicBlock *VPBlockBase::getExitBasicBlock() {
83 VPBlockBase *Block = this;
84 while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
85 Block = Region->getExit();
86 return cast<VPBasicBlock>(Block);
87}
88
89VPBlockBase *VPBlockBase::getEnclosingBlockWithSuccessors() {
90 if (!Successors.empty() || !Parent)
91 return this;
92 assert(Parent->getExit() == this &&((Parent->getExit() == this && "Block w/o successors not the exit of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getExit() == this && \"Block w/o successors not the exit of its parent.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 93, __PRETTY_FUNCTION__))
93 "Block w/o successors not the exit of its parent.")((Parent->getExit() == this && "Block w/o successors not the exit of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getExit() == this && \"Block w/o successors not the exit of its parent.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 93, __PRETTY_FUNCTION__))
;
94 return Parent->getEnclosingBlockWithSuccessors();
95}
96
97VPBlockBase *VPBlockBase::getEnclosingBlockWithPredecessors() {
98 if (!Predecessors.empty() || !Parent)
99 return this;
100 assert(Parent->getEntry() == this &&((Parent->getEntry() == this && "Block w/o predecessors not the entry of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getEntry() == this && \"Block w/o predecessors not the entry of its parent.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 101, __PRETTY_FUNCTION__))
101 "Block w/o predecessors not the entry of its parent.")((Parent->getEntry() == this && "Block w/o predecessors not the entry of its parent."
) ? static_cast<void> (0) : __assert_fail ("Parent->getEntry() == this && \"Block w/o predecessors not the entry of its parent.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 101, __PRETTY_FUNCTION__))
;
102 return Parent->getEnclosingBlockWithPredecessors();
103}
104
105void VPBlockBase::deleteCFG(VPBlockBase *Entry) {
106 SmallVector<VPBlockBase *, 8> Blocks;
107 for (VPBlockBase *Block : depth_first(Entry))
108 Blocks.push_back(Block);
109
110 for (VPBlockBase *Block : Blocks)
111 delete Block;
112}
113
114BasicBlock *
115VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
116 // BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks.
117 // Pred stands for Predessor. Prev stands for Previous - last visited/created.
118 BasicBlock *PrevBB = CFG.PrevBB;
119 BasicBlock *NewBB = BasicBlock::Create(PrevBB->getContext(), getName(),
120 PrevBB->getParent(), CFG.LastBB);
121 LLVM_DEBUG(dbgs() << "LV: created " << NewBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: created " << NewBB->
getName() << '\n'; } } while (false)
;
122
123 // Hook up the new basic block to its predecessors.
124 for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
125 VPBasicBlock *PredVPBB = PredVPBlock->getExitBasicBlock();
126 auto &PredVPSuccessors = PredVPBB->getSuccessors();
127 BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
128
129 // In outer loop vectorization scenario, the predecessor BBlock may not yet
130 // be visited(backedge). Mark the VPBasicBlock for fixup at the end of
131 // vectorization. We do not encounter this case in inner loop vectorization
132 // as we start out by building a loop skeleton with the vector loop header
133 // and latch blocks. As a result, we never enter this function for the
134 // header block in the non VPlan-native path.
135 if (!PredBB) {
136 assert(EnableVPlanNativePath &&((EnableVPlanNativePath && "Unexpected null predecessor in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected null predecessor in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 137, __PRETTY_FUNCTION__))
137 "Unexpected null predecessor in non VPlan-native path")((EnableVPlanNativePath && "Unexpected null predecessor in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected null predecessor in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 137, __PRETTY_FUNCTION__))
;
138 CFG.VPBBsToFix.push_back(PredVPBB);
139 continue;
140 }
141
142 assert(PredBB && "Predecessor basic-block not found building successor.")((PredBB && "Predecessor basic-block not found building successor."
) ? static_cast<void> (0) : __assert_fail ("PredBB && \"Predecessor basic-block not found building successor.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 142, __PRETTY_FUNCTION__))
;
143 auto *PredBBTerminator = PredBB->getTerminator();
144 LLVM_DEBUG(dbgs() << "LV: draw edge from" << PredBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: draw edge from" << PredBB
->getName() << '\n'; } } while (false)
;
145 if (isa<UnreachableInst>(PredBBTerminator)) {
146 assert(PredVPSuccessors.size() == 1 &&((PredVPSuccessors.size() == 1 && "Predecessor ending w/o branch must have single successor."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 1 && \"Predecessor ending w/o branch must have single successor.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 147, __PRETTY_FUNCTION__))
147 "Predecessor ending w/o branch must have single successor.")((PredVPSuccessors.size() == 1 && "Predecessor ending w/o branch must have single successor."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 1 && \"Predecessor ending w/o branch must have single successor.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 147, __PRETTY_FUNCTION__))
;
148 PredBBTerminator->eraseFromParent();
149 BranchInst::Create(NewBB, PredBB);
150 } else {
151 assert(PredVPSuccessors.size() == 2 &&((PredVPSuccessors.size() == 2 && "Predecessor ending with branch must have two successors."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 2 && \"Predecessor ending with branch must have two successors.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 152, __PRETTY_FUNCTION__))
152 "Predecessor ending with branch must have two successors.")((PredVPSuccessors.size() == 2 && "Predecessor ending with branch must have two successors."
) ? static_cast<void> (0) : __assert_fail ("PredVPSuccessors.size() == 2 && \"Predecessor ending with branch must have two successors.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 152, __PRETTY_FUNCTION__))
;
153 unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
154 assert(!PredBBTerminator->getSuccessor(idx) &&((!PredBBTerminator->getSuccessor(idx) && "Trying to reset an existing successor block."
) ? static_cast<void> (0) : __assert_fail ("!PredBBTerminator->getSuccessor(idx) && \"Trying to reset an existing successor block.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 155, __PRETTY_FUNCTION__))
155 "Trying to reset an existing successor block.")((!PredBBTerminator->getSuccessor(idx) && "Trying to reset an existing successor block."
) ? static_cast<void> (0) : __assert_fail ("!PredBBTerminator->getSuccessor(idx) && \"Trying to reset an existing successor block.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 155, __PRETTY_FUNCTION__))
;
156 PredBBTerminator->setSuccessor(idx, NewBB);
157 }
158 }
159 return NewBB;
160}
161
162void VPBasicBlock::execute(VPTransformState *State) {
163 bool Replica = State->Instance &&
164 !(State->Instance->Part == 0 && State->Instance->Lane == 0);
165 VPBasicBlock *PrevVPBB = State->CFG.PrevVPBB;
166 VPBlockBase *SingleHPred = nullptr;
167 BasicBlock *NewBB = State->CFG.PrevBB; // Reuse it if possible.
168
169 // 1. Create an IR basic block, or reuse the last one if possible.
170 // The last IR basic block is reused, as an optimization, in three cases:
171 // A. the first VPBB reuses the loop header BB - when PrevVPBB is null;
172 // B. when the current VPBB has a single (hierarchical) predecessor which
173 // is PrevVPBB and the latter has a single (hierarchical) successor; and
174 // C. when the current VPBB is an entry of a region replica - where PrevVPBB
175 // is the exit of this region from a previous instance, or the predecessor
176 // of this region.
177 if (PrevVPBB && /* A */
178 !((SingleHPred = getSingleHierarchicalPredecessor()) &&
179 SingleHPred->getExitBasicBlock() == PrevVPBB &&
180 PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */
181 !(Replica && getPredecessors().empty())) { /* C */
182 NewBB = createEmptyBasicBlock(State->CFG);
183 State->Builder.SetInsertPoint(NewBB);
184 // Temporarily terminate with unreachable until CFG is rewired.
185 UnreachableInst *Terminator = State->Builder.CreateUnreachable();
186 State->Builder.SetInsertPoint(Terminator);
187 // Register NewBB in its loop. In innermost loops its the same for all BB's.
188 Loop *L = State->LI->getLoopFor(State->CFG.LastBB);
189 L->addBasicBlockToLoop(NewBB, *State->LI);
190 State->CFG.PrevBB = NewBB;
191 }
192
193 // 2. Fill the IR basic block with IR instructions.
194 LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: vectorizing VPBB:" <<
getName() << " in BB:" << NewBB->getName() <<
'\n'; } } while (false)
195 << " in BB:" << NewBB->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: vectorizing VPBB:" <<
getName() << " in BB:" << NewBB->getName() <<
'\n'; } } while (false)
;
196
197 State->CFG.VPBB2IRBB[this] = NewBB;
198 State->CFG.PrevVPBB = this;
199
200 for (VPRecipeBase &Recipe : Recipes)
201 Recipe.execute(*State);
202
203 VPValue *CBV;
204 if (EnableVPlanNativePath && (CBV = getCondBit())) {
205 Value *IRCBV = CBV->getUnderlyingValue();
206 assert(IRCBV && "Unexpected null underlying value for condition bit")((IRCBV && "Unexpected null underlying value for condition bit"
) ? static_cast<void> (0) : __assert_fail ("IRCBV && \"Unexpected null underlying value for condition bit\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 206, __PRETTY_FUNCTION__))
;
207
208 // Condition bit value in a VPBasicBlock is used as the branch selector. In
209 // the VPlan-native path case, since all branches are uniform we generate a
210 // branch instruction using the condition value from vector lane 0 and dummy
211 // successors. The successors are fixed later when the successor blocks are
212 // visited.
213 Value *NewCond = State->Callback.getOrCreateVectorValues(IRCBV, 0);
214 NewCond = State->Builder.CreateExtractElement(NewCond,
215 State->Builder.getInt32(0));
216
217 // Replace the temporary unreachable terminator with the new conditional
218 // branch.
219 auto *CurrentTerminator = NewBB->getTerminator();
220 assert(isa<UnreachableInst>(CurrentTerminator) &&((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional "
"branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 222, __PRETTY_FUNCTION__))
221 "Expected to replace unreachable terminator with conditional "((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional "
"branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 222, __PRETTY_FUNCTION__))
222 "branch.")((isa<UnreachableInst>(CurrentTerminator) && "Expected to replace unreachable terminator with conditional "
"branch.") ? static_cast<void> (0) : __assert_fail ("isa<UnreachableInst>(CurrentTerminator) && \"Expected to replace unreachable terminator with conditional \" \"branch.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 222, __PRETTY_FUNCTION__))
;
223 auto *CondBr = BranchInst::Create(NewBB, nullptr, NewCond);
224 CondBr->setSuccessor(0, nullptr);
225 ReplaceInstWithInst(CurrentTerminator, CondBr);
226 }
227
228 LLVM_DEBUG(dbgs() << "LV: filled BB:" << *NewBB)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: filled BB:" << *NewBB
; } } while (false)
;
229}
230
231void VPRegionBlock::execute(VPTransformState *State) {
232 ReversePostOrderTraversal<VPBlockBase *> RPOT(Entry);
233
234 if (!isReplicator()) {
235 // Visit the VPBlocks connected to "this", starting from it.
236 for (VPBlockBase *Block : RPOT) {
237 if (EnableVPlanNativePath) {
238 // The inner loop vectorization path does not represent loop preheader
239 // and exit blocks as part of the VPlan. In the VPlan-native path, skip
240 // vectorizing loop preheader block. In future, we may replace this
241 // check with the check for loop preheader.
242 if (Block->getNumPredecessors() == 0)
243 continue;
244
245 // Skip vectorizing loop exit block. In future, we may replace this
246 // check with the check for loop exit.
247 if (Block->getNumSuccessors() == 0)
248 continue;
249 }
250
251 LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: VPBlock in RPO " << Block
->getName() << '\n'; } } while (false)
;
252 Block->execute(State);
253 }
254 return;
255 }
256
257 assert(!State->Instance && "Replicating a Region with non-null instance.")((!State->Instance && "Replicating a Region with non-null instance."
) ? static_cast<void> (0) : __assert_fail ("!State->Instance && \"Replicating a Region with non-null instance.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 257, __PRETTY_FUNCTION__))
;
258
259 // Enter replicating mode.
260 State->Instance = {0, 0};
261
262 for (unsigned Part = 0, UF = State->UF; Part < UF; ++Part) {
263 State->Instance->Part = Part;
264 for (unsigned Lane = 0, VF = State->VF; Lane < VF; ++Lane) {
265 State->Instance->Lane = Lane;
266 // Visit the VPBlocks connected to \p this, starting from it.
267 for (VPBlockBase *Block : RPOT) {
268 LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("vplan")) { dbgs() << "LV: VPBlock in RPO " << Block
->getName() << '\n'; } } while (false)
;
269 Block->execute(State);
270 }
271 }
272 }
273
274 // Exit replicating mode.
275 State->Instance.reset();
276}
277
278void VPRecipeBase::insertBefore(VPRecipeBase *InsertPos) {
279 Parent = InsertPos->getParent();
280 Parent->getRecipeList().insert(InsertPos->getIterator(), this);
281}
282
283iplist<VPRecipeBase>::iterator VPRecipeBase::eraseFromParent() {
284 return getParent()->getRecipeList().erase(getIterator());
285}
286
287void VPInstruction::generateInstruction(VPTransformState &State,
288 unsigned Part) {
289 IRBuilder<> &Builder = State.Builder;
290
291 if (Instruction::isBinaryOp(getOpcode())) {
292 Value *A = State.get(getOperand(0), Part);
293 Value *B = State.get(getOperand(1), Part);
294 Value *V = Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B);
295 State.set(this, V, Part);
296 return;
297 }
298
299 switch (getOpcode()) {
300 case VPInstruction::Not: {
301 Value *A = State.get(getOperand(0), Part);
302 Value *V = Builder.CreateNot(A);
303 State.set(this, V, Part);
304 break;
305 }
306 case VPInstruction::ICmpULE: {
307 Value *IV = State.get(getOperand(0), Part);
308 Value *TC = State.get(getOperand(1), Part);
309 Value *V = Builder.CreateICmpULE(IV, TC);
310 State.set(this, V, Part);
311 break;
312 }
313 default:
314 llvm_unreachable("Unsupported opcode for instruction")::llvm::llvm_unreachable_internal("Unsupported opcode for instruction"
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 314)
;
315 }
316}
317
318void VPInstruction::execute(VPTransformState &State) {
319 assert(!State.Instance && "VPInstruction executing an Instance")((!State.Instance && "VPInstruction executing an Instance"
) ? static_cast<void> (0) : __assert_fail ("!State.Instance && \"VPInstruction executing an Instance\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 319, __PRETTY_FUNCTION__))
;
320 for (unsigned Part = 0; Part < State.UF; ++Part)
321 generateInstruction(State, Part);
322}
323
324void VPInstruction::print(raw_ostream &O, const Twine &Indent) const {
325 O << " +\n" << Indent << "\"EMIT ";
326 print(O);
327 O << "\\l\"";
328}
329
330void VPInstruction::print(raw_ostream &O) const {
331 printAsOperand(O);
332 O << " = ";
333
334 switch (getOpcode()) {
335 case VPInstruction::Not:
336 O << "not";
337 break;
338 case VPInstruction::ICmpULE:
339 O << "icmp ule";
340 break;
341 case VPInstruction::SLPLoad:
342 O << "combined load";
343 break;
344 case VPInstruction::SLPStore:
345 O << "combined store";
346 break;
347 default:
348 O << Instruction::getOpcodeName(getOpcode());
349 }
350
351 for (const VPValue *Operand : operands()) {
352 O << " ";
353 Operand->printAsOperand(O);
354 }
355}
356
357/// Generate the code inside the body of the vectorized loop. Assumes a single
358/// LoopVectorBody basic-block was created for this. Introduce additional
359/// basic-blocks as needed, and fill them all.
360void VPlan::execute(VPTransformState *State) {
361 // -1. Check if the backedge taken count is needed, and if so build it.
362 if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
363 Value *TC = State->TripCount;
364 IRBuilder<> Builder(State->CFG.PrevBB->getTerminator());
365 auto *TCMO = Builder.CreateSub(TC, ConstantInt::get(TC->getType(), 1),
366 "trip.count.minus.1");
367 Value2VPValue[TCMO] = BackedgeTakenCount;
368 }
369
370 // 0. Set the reverse mapping from VPValues to Values for code generation.
371 for (auto &Entry : Value2VPValue)
372 State->VPValue2Value[Entry.second] = Entry.first;
373
374 BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB;
375 BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor();
376 assert(VectorHeaderBB && "Loop preheader does not have a single successor.")((VectorHeaderBB && "Loop preheader does not have a single successor."
) ? static_cast<void> (0) : __assert_fail ("VectorHeaderBB && \"Loop preheader does not have a single successor.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 376, __PRETTY_FUNCTION__))
;
377 BasicBlock *VectorLatchBB = VectorHeaderBB;
Value stored to 'VectorLatchBB' during its initialization is never read
378
379 // 1. Make room to generate basic-blocks inside loop body if needed.
380 VectorLatchBB = VectorHeaderBB->splitBasicBlock(
381 VectorHeaderBB->getFirstInsertionPt(), "vector.body.latch");
382 Loop *L = State->LI->getLoopFor(VectorHeaderBB);
383 L->addBasicBlockToLoop(VectorLatchBB, *State->LI);
384 // Remove the edge between Header and Latch to allow other connections.
385 // Temporarily terminate with unreachable until CFG is rewired.
386 // Note: this asserts the generated code's assumption that
387 // getFirstInsertionPt() can be dereferenced into an Instruction.
388 VectorHeaderBB->getTerminator()->eraseFromParent();
389 State->Builder.SetInsertPoint(VectorHeaderBB);
390 UnreachableInst *Terminator = State->Builder.CreateUnreachable();
391 State->Builder.SetInsertPoint(Terminator);
392
393 // 2. Generate code in loop body.
394 State->CFG.PrevVPBB = nullptr;
395 State->CFG.PrevBB = VectorHeaderBB;
396 State->CFG.LastBB = VectorLatchBB;
397
398 for (VPBlockBase *Block : depth_first(Entry))
399 Block->execute(State);
400
401 // Setup branch terminator successors for VPBBs in VPBBsToFix based on
402 // VPBB's successors.
403 for (auto VPBB : State->CFG.VPBBsToFix) {
404 assert(EnableVPlanNativePath &&((EnableVPlanNativePath && "Unexpected VPBBsToFix in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected VPBBsToFix in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 405, __PRETTY_FUNCTION__))
405 "Unexpected VPBBsToFix in non VPlan-native path")((EnableVPlanNativePath && "Unexpected VPBBsToFix in non VPlan-native path"
) ? static_cast<void> (0) : __assert_fail ("EnableVPlanNativePath && \"Unexpected VPBBsToFix in non VPlan-native path\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 405, __PRETTY_FUNCTION__))
;
406 BasicBlock *BB = State->CFG.VPBB2IRBB[VPBB];
407 assert(BB && "Unexpected null basic block for VPBB")((BB && "Unexpected null basic block for VPBB") ? static_cast
<void> (0) : __assert_fail ("BB && \"Unexpected null basic block for VPBB\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 407, __PRETTY_FUNCTION__))
;
408
409 unsigned Idx = 0;
410 auto *BBTerminator = BB->getTerminator();
411
412 for (VPBlockBase *SuccVPBlock : VPBB->getHierarchicalSuccessors()) {
413 VPBasicBlock *SuccVPBB = SuccVPBlock->getEntryBasicBlock();
414 BBTerminator->setSuccessor(Idx, State->CFG.VPBB2IRBB[SuccVPBB]);
415 ++Idx;
416 }
417 }
418
419 // 3. Merge the temporary latch created with the last basic-block filled.
420 BasicBlock *LastBB = State->CFG.PrevBB;
421 // Connect LastBB to VectorLatchBB to facilitate their merge.
422 assert((EnableVPlanNativePath ||(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB
->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable"
) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 424, __PRETTY_FUNCTION__))
423 isa<UnreachableInst>(LastBB->getTerminator())) &&(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB
->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable"
) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 424, __PRETTY_FUNCTION__))
424 "Expected InnerLoop VPlan CFG to terminate with unreachable")(((EnableVPlanNativePath || isa<UnreachableInst>(LastBB
->getTerminator())) && "Expected InnerLoop VPlan CFG to terminate with unreachable"
) ? static_cast<void> (0) : __assert_fail ("(EnableVPlanNativePath || isa<UnreachableInst>(LastBB->getTerminator())) && \"Expected InnerLoop VPlan CFG to terminate with unreachable\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 424, __PRETTY_FUNCTION__))
;
425 assert((!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) &&(((!EnableVPlanNativePath || isa<BranchInst>(LastBB->
getTerminator())) && "Expected VPlan CFG to terminate with branch in NativePath"
) ? static_cast<void> (0) : __assert_fail ("(!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) && \"Expected VPlan CFG to terminate with branch in NativePath\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 426, __PRETTY_FUNCTION__))
426 "Expected VPlan CFG to terminate with branch in NativePath")(((!EnableVPlanNativePath || isa<BranchInst>(LastBB->
getTerminator())) && "Expected VPlan CFG to terminate with branch in NativePath"
) ? static_cast<void> (0) : __assert_fail ("(!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) && \"Expected VPlan CFG to terminate with branch in NativePath\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 426, __PRETTY_FUNCTION__))
;
427 LastBB->getTerminator()->eraseFromParent();
428 BranchInst::Create(VectorLatchBB, LastBB);
429
430 // Merge LastBB with Latch.
431 bool Merged = MergeBlockIntoPredecessor(VectorLatchBB, nullptr, State->LI);
432 (void)Merged;
433 assert(Merged && "Could not merge last basic block with latch.")((Merged && "Could not merge last basic block with latch."
) ? static_cast<void> (0) : __assert_fail ("Merged && \"Could not merge last basic block with latch.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 433, __PRETTY_FUNCTION__))
;
434 VectorLatchBB = LastBB;
435
436 // We do not attempt to preserve DT for outer loop vectorization currently.
437 if (!EnableVPlanNativePath)
438 updateDominatorTree(State->DT, VectorPreHeaderBB, VectorLatchBB);
439}
440
441void VPlan::updateDominatorTree(DominatorTree *DT, BasicBlock *LoopPreHeaderBB,
442 BasicBlock *LoopLatchBB) {
443 BasicBlock *LoopHeaderBB = LoopPreHeaderBB->getSingleSuccessor();
444 assert(LoopHeaderBB && "Loop preheader does not have a single successor.")((LoopHeaderBB && "Loop preheader does not have a single successor."
) ? static_cast<void> (0) : __assert_fail ("LoopHeaderBB && \"Loop preheader does not have a single successor.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 444, __PRETTY_FUNCTION__))
;
445 DT->addNewBlock(LoopHeaderBB, LoopPreHeaderBB);
446 // The vector body may be more than a single basic-block by this point.
447 // Update the dominator tree information inside the vector body by propagating
448 // it from header to latch, expecting only triangular control-flow, if any.
449 BasicBlock *PostDomSucc = nullptr;
450 for (auto *BB = LoopHeaderBB; BB != LoopLatchBB; BB = PostDomSucc) {
451 // Get the list of successors of this block.
452 std::vector<BasicBlock *> Succs(succ_begin(BB), succ_end(BB));
453 assert(Succs.size() <= 2 &&((Succs.size() <= 2 && "Basic block in vector loop has more than 2 successors."
) ? static_cast<void> (0) : __assert_fail ("Succs.size() <= 2 && \"Basic block in vector loop has more than 2 successors.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 454, __PRETTY_FUNCTION__))
454 "Basic block in vector loop has more than 2 successors.")((Succs.size() <= 2 && "Basic block in vector loop has more than 2 successors."
) ? static_cast<void> (0) : __assert_fail ("Succs.size() <= 2 && \"Basic block in vector loop has more than 2 successors.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 454, __PRETTY_FUNCTION__))
;
455 PostDomSucc = Succs[0];
456 if (Succs.size() == 1) {
457 assert(PostDomSucc->getSinglePredecessor() &&((PostDomSucc->getSinglePredecessor() && "PostDom successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->getSinglePredecessor() && \"PostDom successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 458, __PRETTY_FUNCTION__))
458 "PostDom successor has more than one predecessor.")((PostDomSucc->getSinglePredecessor() && "PostDom successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->getSinglePredecessor() && \"PostDom successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 458, __PRETTY_FUNCTION__))
;
459 DT->addNewBlock(PostDomSucc, BB);
460 continue;
461 }
462 BasicBlock *InterimSucc = Succs[1];
463 if (PostDomSucc->getSingleSuccessor() == InterimSucc) {
464 PostDomSucc = Succs[1];
465 InterimSucc = Succs[0];
466 }
467 assert(InterimSucc->getSingleSuccessor() == PostDomSucc &&((InterimSucc->getSingleSuccessor() == PostDomSucc &&
"One successor of a basic block does not lead to the other."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSingleSuccessor() == PostDomSucc && \"One successor of a basic block does not lead to the other.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 468, __PRETTY_FUNCTION__))
468 "One successor of a basic block does not lead to the other.")((InterimSucc->getSingleSuccessor() == PostDomSucc &&
"One successor of a basic block does not lead to the other."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSingleSuccessor() == PostDomSucc && \"One successor of a basic block does not lead to the other.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 468, __PRETTY_FUNCTION__))
;
469 assert(InterimSucc->getSinglePredecessor() &&((InterimSucc->getSinglePredecessor() && "Interim successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSinglePredecessor() && \"Interim successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 470, __PRETTY_FUNCTION__))
470 "Interim successor has more than one predecessor.")((InterimSucc->getSinglePredecessor() && "Interim successor has more than one predecessor."
) ? static_cast<void> (0) : __assert_fail ("InterimSucc->getSinglePredecessor() && \"Interim successor has more than one predecessor.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 470, __PRETTY_FUNCTION__))
;
471 assert(PostDomSucc->hasNPredecessors(2) &&((PostDomSucc->hasNPredecessors(2) && "PostDom successor has more than two predecessors."
) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->hasNPredecessors(2) && \"PostDom successor has more than two predecessors.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 472, __PRETTY_FUNCTION__))
472 "PostDom successor has more than two predecessors.")((PostDomSucc->hasNPredecessors(2) && "PostDom successor has more than two predecessors."
) ? static_cast<void> (0) : __assert_fail ("PostDomSucc->hasNPredecessors(2) && \"PostDom successor has more than two predecessors.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 472, __PRETTY_FUNCTION__))
;
473 DT->addNewBlock(InterimSucc, BB);
474 DT->addNewBlock(PostDomSucc, BB);
475 }
476}
477
478const Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
479 return (isa<VPRegionBlock>(Block) ? "cluster_N" : "N") +
480 Twine(getOrCreateBID(Block));
481}
482
483const Twine VPlanPrinter::getOrCreateName(const VPBlockBase *Block) {
484 const std::string &Name = Block->getName();
485 if (!Name.empty())
486 return Name;
487 return "VPB" + Twine(getOrCreateBID(Block));
488}
489
490void VPlanPrinter::dump() {
491 Depth = 1;
492 bumpIndent(0);
493 OS << "digraph VPlan {\n";
494 OS << "graph [labelloc=t, fontsize=30; label=\"Vectorization Plan";
495 if (!Plan.getName().empty())
496 OS << "\\n" << DOT::EscapeString(Plan.getName());
497 if (!Plan.Value2VPValue.empty() || Plan.BackedgeTakenCount) {
498 OS << ", where:";
499 if (Plan.BackedgeTakenCount)
500 OS << "\\n"
501 << *Plan.getOrCreateBackedgeTakenCount() << " := BackedgeTakenCount";
502 for (auto Entry : Plan.Value2VPValue) {
503 OS << "\\n" << *Entry.second;
504 OS << DOT::EscapeString(" := ");
505 Entry.first->printAsOperand(OS, false);
506 }
507 }
508 OS << "\"]\n";
509 OS << "node [shape=rect, fontname=Courier, fontsize=30]\n";
510 OS << "edge [fontname=Courier, fontsize=30]\n";
511 OS << "compound=true\n";
512
513 for (VPBlockBase *Block : depth_first(Plan.getEntry()))
514 dumpBlock(Block);
515
516 OS << "}\n";
517}
518
519void VPlanPrinter::dumpBlock(const VPBlockBase *Block) {
520 if (const VPBasicBlock *BasicBlock = dyn_cast<VPBasicBlock>(Block))
521 dumpBasicBlock(BasicBlock);
522 else if (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
523 dumpRegion(Region);
524 else
525 llvm_unreachable("Unsupported kind of VPBlock.")::llvm::llvm_unreachable_internal("Unsupported kind of VPBlock."
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 525)
;
526}
527
528void VPlanPrinter::drawEdge(const VPBlockBase *From, const VPBlockBase *To,
529 bool Hidden, const Twine &Label) {
530 // Due to "dot" we print an edge between two regions as an edge between the
531 // exit basic block and the entry basic of the respective regions.
532 const VPBlockBase *Tail = From->getExitBasicBlock();
533 const VPBlockBase *Head = To->getEntryBasicBlock();
534 OS << Indent << getUID(Tail) << " -> " << getUID(Head);
535 OS << " [ label=\"" << Label << '\"';
536 if (Tail != From)
537 OS << " ltail=" << getUID(From);
538 if (Head != To)
539 OS << " lhead=" << getUID(To);
540 if (Hidden)
541 OS << "; splines=none";
542 OS << "]\n";
543}
544
545void VPlanPrinter::dumpEdges(const VPBlockBase *Block) {
546 auto &Successors = Block->getSuccessors();
547 if (Successors.size() == 1)
548 drawEdge(Block, Successors.front(), false, "");
549 else if (Successors.size() == 2) {
550 drawEdge(Block, Successors.front(), false, "T");
551 drawEdge(Block, Successors.back(), false, "F");
552 } else {
553 unsigned SuccessorNumber = 0;
554 for (auto *Successor : Successors)
555 drawEdge(Block, Successor, false, Twine(SuccessorNumber++));
556 }
557}
558
559void VPlanPrinter::dumpBasicBlock(const VPBasicBlock *BasicBlock) {
560 OS << Indent << getUID(BasicBlock) << " [label =\n";
561 bumpIndent(1);
562 OS << Indent << "\"" << DOT::EscapeString(BasicBlock->getName()) << ":\\n\"";
563 bumpIndent(1);
564 for (const VPRecipeBase &Recipe : *BasicBlock)
565 Recipe.print(OS, Indent);
566
567 // Dump the condition bit.
568 const VPValue *CBV = BasicBlock->getCondBit();
569 if (CBV) {
570 OS << " +\n" << Indent << " \"CondBit: ";
571 if (const VPInstruction *CBI = dyn_cast<VPInstruction>(CBV)) {
572 CBI->printAsOperand(OS);
573 OS << " (" << DOT::EscapeString(CBI->getParent()->getName()) << ")\\l\"";
574 } else {
575 CBV->printAsOperand(OS);
576 OS << "\"";
577 }
578 }
579
580 bumpIndent(-2);
581 OS << "\n" << Indent << "]\n";
582 dumpEdges(BasicBlock);
583}
584
585void VPlanPrinter::dumpRegion(const VPRegionBlock *Region) {
586 OS << Indent << "subgraph " << getUID(Region) << " {\n";
587 bumpIndent(1);
588 OS << Indent << "fontname=Courier\n"
589 << Indent << "label=\""
590 << DOT::EscapeString(Region->isReplicator() ? "<xVFxUF> " : "<x1> ")
591 << DOT::EscapeString(Region->getName()) << "\"\n";
592 // Dump the blocks of the region.
593 assert(Region->getEntry() && "Region contains no inner blocks.")((Region->getEntry() && "Region contains no inner blocks."
) ? static_cast<void> (0) : __assert_fail ("Region->getEntry() && \"Region contains no inner blocks.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 593, __PRETTY_FUNCTION__))
;
594 for (const VPBlockBase *Block : depth_first(Region->getEntry()))
595 dumpBlock(Block);
596 bumpIndent(-1);
597 OS << Indent << "}\n";
598 dumpEdges(Region);
599}
600
601void VPlanPrinter::printAsIngredient(raw_ostream &O, Value *V) {
602 std::string IngredientString;
603 raw_string_ostream RSO(IngredientString);
604 if (auto *Inst = dyn_cast<Instruction>(V)) {
605 if (!Inst->getType()->isVoidTy()) {
606 Inst->printAsOperand(RSO, false);
607 RSO << " = ";
608 }
609 RSO << Inst->getOpcodeName() << " ";
610 unsigned E = Inst->getNumOperands();
611 if (E > 0) {
612 Inst->getOperand(0)->printAsOperand(RSO, false);
613 for (unsigned I = 1; I < E; ++I)
614 Inst->getOperand(I)->printAsOperand(RSO << ", ", false);
615 }
616 } else // !Inst
617 V->printAsOperand(RSO, false);
618 RSO.flush();
619 O << DOT::EscapeString(IngredientString);
620}
621
622void VPWidenRecipe::print(raw_ostream &O, const Twine &Indent) const {
623 O << " +\n" << Indent << "\"WIDEN\\l\"";
624 for (auto &Instr : make_range(Begin, End))
625 O << " +\n" << Indent << "\" " << VPlanIngredient(&Instr) << "\\l\"";
626}
627
628void VPWidenIntOrFpInductionRecipe::print(raw_ostream &O,
629 const Twine &Indent) const {
630 O << " +\n" << Indent << "\"WIDEN-INDUCTION";
631 if (Trunc) {
632 O << "\\l\"";
633 O << " +\n" << Indent << "\" " << VPlanIngredient(IV) << "\\l\"";
634 O << " +\n" << Indent << "\" " << VPlanIngredient(Trunc) << "\\l\"";
635 } else
636 O << " " << VPlanIngredient(IV) << "\\l\"";
637}
638
639void VPWidenPHIRecipe::print(raw_ostream &O, const Twine &Indent) const {
640 O << " +\n" << Indent << "\"WIDEN-PHI " << VPlanIngredient(Phi) << "\\l\"";
641}
642
643void VPBlendRecipe::print(raw_ostream &O, const Twine &Indent) const {
644 O << " +\n" << Indent << "\"BLEND ";
645 Phi->printAsOperand(O, false);
646 O << " =";
647 if (!User) {
648 // Not a User of any mask: not really blending, this is a
649 // single-predecessor phi.
650 O << " ";
651 Phi->getIncomingValue(0)->printAsOperand(O, false);
652 } else {
653 for (unsigned I = 0, E = User->getNumOperands(); I < E; ++I) {
654 O << " ";
655 Phi->getIncomingValue(I)->printAsOperand(O, false);
656 O << "/";
657 User->getOperand(I)->printAsOperand(O);
658 }
659 }
660 O << "\\l\"";
661}
662
663void VPReplicateRecipe::print(raw_ostream &O, const Twine &Indent) const {
664 O << " +\n"
665 << Indent << "\"" << (IsUniform ? "CLONE " : "REPLICATE ")
666 << VPlanIngredient(Ingredient);
667 if (AlsoPack)
668 O << " (S->V)";
669 O << "\\l\"";
670}
671
672void VPPredInstPHIRecipe::print(raw_ostream &O, const Twine &Indent) const {
673 O << " +\n"
674 << Indent << "\"PHI-PREDICATED-INSTRUCTION " << VPlanIngredient(PredInst)
675 << "\\l\"";
676}
677
678void VPWidenMemoryInstructionRecipe::print(raw_ostream &O,
679 const Twine &Indent) const {
680 O << " +\n" << Indent << "\"WIDEN " << VPlanIngredient(&Instr);
681 if (User) {
682 O << ", ";
683 User->getOperand(0)->printAsOperand(O);
684 }
685 O << "\\l\"";
686}
687
688template void DomTreeBuilder::Calculate<VPDominatorTree>(VPDominatorTree &DT);
689
690void VPValue::replaceAllUsesWith(VPValue *New) {
691 for (VPUser *User : users())
692 for (unsigned I = 0, E = User->getNumOperands(); I < E; ++I)
693 if (User->getOperand(I) == this)
694 User->setOperand(I, New);
695}
696
697void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
698 Old2NewTy &Old2New,
699 InterleavedAccessInfo &IAI) {
700 ReversePostOrderTraversal<VPBlockBase *> RPOT(Region->getEntry());
701 for (VPBlockBase *Base : RPOT) {
702 visitBlock(Base, Old2New, IAI);
703 }
704}
705
706void VPInterleavedAccessInfo::visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
707 InterleavedAccessInfo &IAI) {
708 if (VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(Block)) {
709 for (VPRecipeBase &VPI : *VPBB) {
710 assert(isa<VPInstruction>(&VPI) && "Can only handle VPInstructions")((isa<VPInstruction>(&VPI) && "Can only handle VPInstructions"
) ? static_cast<void> (0) : __assert_fail ("isa<VPInstruction>(&VPI) && \"Can only handle VPInstructions\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 710, __PRETTY_FUNCTION__))
;
711 auto *VPInst = cast<VPInstruction>(&VPI);
712 auto *Inst = cast<Instruction>(VPInst->getUnderlyingValue());
713 auto *IG = IAI.getInterleaveGroup(Inst);
714 if (!IG)
715 continue;
716
717 auto NewIGIter = Old2New.find(IG);
718 if (NewIGIter == Old2New.end())
719 Old2New[IG] = new InterleaveGroup<VPInstruction>(
720 IG->getFactor(), IG->isReverse(), IG->getAlignment());
721
722 if (Inst == IG->getInsertPos())
723 Old2New[IG]->setInsertPos(VPInst);
724
725 InterleaveGroupMap[VPInst] = Old2New[IG];
726 InterleaveGroupMap[VPInst]->insertMember(
727 VPInst, IG->getIndex(Inst),
728 IG->isReverse() ? (-1) * int(IG->getFactor()) : IG->getFactor());
729 }
730 } else if (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
731 visitRegion(Region, Old2New, IAI);
732 else
733 llvm_unreachable("Unsupported kind of VPBlock.")::llvm::llvm_unreachable_internal("Unsupported kind of VPBlock."
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Vectorize/VPlan.cpp"
, 733)
;
734}
735
736VPInterleavedAccessInfo::VPInterleavedAccessInfo(VPlan &Plan,
737 InterleavedAccessInfo &IAI) {
738 Old2NewTy Old2New;
739 visitRegion(cast<VPRegionBlock>(Plan.getEntry()), Old2New, IAI);
740}