Bug Summary

File:lib/Transforms/Scalar/ConstantHoisting.cpp
Warning:line 335, column 25
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name ConstantHoisting.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar -I /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn350071/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn350071/build-llvm/lib/Transforms/Scalar -fdebug-prefix-map=/build/llvm-toolchain-snapshot-8~svn350071=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-12-27-042839-1215-1 -x c++ /build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp -faddrsig
1//===- ConstantHoisting.cpp - Prepare code for expensive constants --------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This pass identifies expensive constants to hoist and coalesces them to
11// better prepare it for SelectionDAG-based code generation. This works around
12// the limitations of the basic-block-at-a-time approach.
13//
14// First it scans all instructions for integer constants and calculates its
15// cost. If the constant can be folded into the instruction (the cost is
16// TCC_Free) or the cost is just a simple operation (TCC_BASIC), then we don't
17// consider it expensive and leave it alone. This is the default behavior and
18// the default implementation of getIntImmCost will always return TCC_Free.
19//
20// If the cost is more than TCC_BASIC, then the integer constant can't be folded
21// into the instruction and it might be beneficial to hoist the constant.
22// Similar constants are coalesced to reduce register pressure and
23// materialization code.
24//
25// When a constant is hoisted, it is also hidden behind a bitcast to force it to
26// be live-out of the basic block. Otherwise the constant would be just
27// duplicated and each basic block would have its own copy in the SelectionDAG.
28// The SelectionDAG recognizes such constants as opaque and doesn't perform
29// certain transformations on them, which would create a new expensive constant.
30//
31// This optimization is only applied to integer constants in instructions and
32// simple (this means not nested) constant cast expressions. For example:
33// %0 = load i64* inttoptr (i64 big_constant to i64*)
34//===----------------------------------------------------------------------===//
35
36#include "llvm/Transforms/Scalar/ConstantHoisting.h"
37#include "llvm/ADT/APInt.h"
38#include "llvm/ADT/DenseMap.h"
39#include "llvm/ADT/None.h"
40#include "llvm/ADT/Optional.h"
41#include "llvm/ADT/SmallPtrSet.h"
42#include "llvm/ADT/SmallVector.h"
43#include "llvm/ADT/Statistic.h"
44#include "llvm/Analysis/BlockFrequencyInfo.h"
45#include "llvm/Analysis/TargetTransformInfo.h"
46#include "llvm/Transforms/Utils/Local.h"
47#include "llvm/IR/BasicBlock.h"
48#include "llvm/IR/Constants.h"
49#include "llvm/IR/DebugInfoMetadata.h"
50#include "llvm/IR/Dominators.h"
51#include "llvm/IR/Function.h"
52#include "llvm/IR/InstrTypes.h"
53#include "llvm/IR/Instruction.h"
54#include "llvm/IR/Instructions.h"
55#include "llvm/IR/IntrinsicInst.h"
56#include "llvm/IR/Value.h"
57#include "llvm/Pass.h"
58#include "llvm/Support/BlockFrequency.h"
59#include "llvm/Support/Casting.h"
60#include "llvm/Support/CommandLine.h"
61#include "llvm/Support/Debug.h"
62#include "llvm/Support/raw_ostream.h"
63#include "llvm/Transforms/Scalar.h"
64#include <algorithm>
65#include <cassert>
66#include <cstdint>
67#include <iterator>
68#include <tuple>
69#include <utility>
70
71using namespace llvm;
72using namespace consthoist;
73
74#define DEBUG_TYPE"consthoist" "consthoist"
75
76STATISTIC(NumConstantsHoisted, "Number of constants hoisted")static llvm::Statistic NumConstantsHoisted = {"consthoist", "NumConstantsHoisted"
, "Number of constants hoisted", {0}, {false}}
;
77STATISTIC(NumConstantsRebased, "Number of constants rebased")static llvm::Statistic NumConstantsRebased = {"consthoist", "NumConstantsRebased"
, "Number of constants rebased", {0}, {false}}
;
78
79static cl::opt<bool> ConstHoistWithBlockFrequency(
80 "consthoist-with-block-frequency", cl::init(true), cl::Hidden,
81 cl::desc("Enable the use of the block frequency analysis to reduce the "
82 "chance to execute const materialization more frequently than "
83 "without hoisting."));
84
85static cl::opt<bool> ConstHoistGEP(
86 "consthoist-gep", cl::init(false), cl::Hidden,
87 cl::desc("Try hoisting constant gep expressions"));
88
89static cl::opt<unsigned>
90MinNumOfDependentToRebase("consthoist-min-num-to-rebase",
91 cl::desc("Do not rebase if number of dependent constants of a Base is less "
92 "than this number."),
93 cl::init(0), cl::Hidden);
94
95namespace {
96
97/// The constant hoisting pass.
98class ConstantHoistingLegacyPass : public FunctionPass {
99public:
100 static char ID; // Pass identification, replacement for typeid
101
102 ConstantHoistingLegacyPass() : FunctionPass(ID) {
103 initializeConstantHoistingLegacyPassPass(*PassRegistry::getPassRegistry());
104 }
105
106 bool runOnFunction(Function &Fn) override;
107
108 StringRef getPassName() const override { return "Constant Hoisting"; }
109
110 void getAnalysisUsage(AnalysisUsage &AU) const override {
111 AU.setPreservesCFG();
112 if (ConstHoistWithBlockFrequency)
113 AU.addRequired<BlockFrequencyInfoWrapperPass>();
114 AU.addRequired<DominatorTreeWrapperPass>();
115 AU.addRequired<TargetTransformInfoWrapperPass>();
116 }
117
118 void releaseMemory() override { Impl.releaseMemory(); }
119
120private:
121 ConstantHoistingPass Impl;
122};
123
124} // end anonymous namespace
125
126char ConstantHoistingLegacyPass::ID = 0;
127
128INITIALIZE_PASS_BEGIN(ConstantHoistingLegacyPass, "consthoist",static void *initializeConstantHoistingLegacyPassPassOnce(PassRegistry
&Registry) {
129 "Constant Hoisting", false, false)static void *initializeConstantHoistingLegacyPassPassOnce(PassRegistry
&Registry) {
130INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)initializeBlockFrequencyInfoWrapperPassPass(Registry);
131INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)initializeDominatorTreeWrapperPassPass(Registry);
132INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)initializeTargetTransformInfoWrapperPassPass(Registry);
133INITIALIZE_PASS_END(ConstantHoistingLegacyPass, "consthoist",PassInfo *PI = new PassInfo( "Constant Hoisting", "consthoist"
, &ConstantHoistingLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<ConstantHoistingLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeConstantHoistingLegacyPassPassFlag; void
llvm::initializeConstantHoistingLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeConstantHoistingLegacyPassPassFlag
, initializeConstantHoistingLegacyPassPassOnce, std::ref(Registry
)); }
134 "Constant Hoisting", false, false)PassInfo *PI = new PassInfo( "Constant Hoisting", "consthoist"
, &ConstantHoistingLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<ConstantHoistingLegacyPass>), false, false
); Registry.registerPass(*PI, true); return PI; } static llvm
::once_flag InitializeConstantHoistingLegacyPassPassFlag; void
llvm::initializeConstantHoistingLegacyPassPass(PassRegistry &
Registry) { llvm::call_once(InitializeConstantHoistingLegacyPassPassFlag
, initializeConstantHoistingLegacyPassPassOnce, std::ref(Registry
)); }
135
136FunctionPass *llvm::createConstantHoistingPass() {
137 return new ConstantHoistingLegacyPass();
138}
139
140/// Perform the constant hoisting optimization for the given function.
141bool ConstantHoistingLegacyPass::runOnFunction(Function &Fn) {
142 if (skipFunction(Fn))
143 return false;
144
145 LLVM_DEBUG(dbgs() << "********** Begin Constant Hoisting **********\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "********** Begin Constant Hoisting **********\n"
; } } while (false)
;
146 LLVM_DEBUG(dbgs() << "********** Function: " << Fn.getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "********** Function: " <<
Fn.getName() << '\n'; } } while (false)
;
147
148 bool MadeChange =
149 Impl.runImpl(Fn, getAnalysis<TargetTransformInfoWrapperPass>().getTTI(Fn),
150 getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
151 ConstHoistWithBlockFrequency
152 ? &getAnalysis<BlockFrequencyInfoWrapperPass>().getBFI()
153 : nullptr,
154 Fn.getEntryBlock());
155
156 if (MadeChange) {
157 LLVM_DEBUG(dbgs() << "********** Function after Constant Hoisting: "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "********** Function after Constant Hoisting: "
<< Fn.getName() << '\n'; } } while (false)
158 << Fn.getName() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "********** Function after Constant Hoisting: "
<< Fn.getName() << '\n'; } } while (false)
;
159 LLVM_DEBUG(dbgs() << Fn)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << Fn; } } while (false)
;
160 }
161 LLVM_DEBUG(dbgs() << "********** End Constant Hoisting **********\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "********** End Constant Hoisting **********\n"
; } } while (false)
;
162
163 return MadeChange;
164}
165
166/// Find the constant materialization insertion point.
167Instruction *ConstantHoistingPass::findMatInsertPt(Instruction *Inst,
168 unsigned Idx) const {
169 // If the operand is a cast instruction, then we have to materialize the
170 // constant before the cast instruction.
171 if (Idx != ~0U) {
172 Value *Opnd = Inst->getOperand(Idx);
173 if (auto CastInst = dyn_cast<Instruction>(Opnd))
174 if (CastInst->isCast())
175 return CastInst;
176 }
177
178 // The simple and common case. This also includes constant expressions.
179 if (!isa<PHINode>(Inst) && !Inst->isEHPad())
180 return Inst;
181
182 // We can't insert directly before a phi node or an eh pad. Insert before
183 // the terminator of the incoming or dominating block.
184 assert(Entry != Inst->getParent() && "PHI or landing pad in entry block!")((Entry != Inst->getParent() && "PHI or landing pad in entry block!"
) ? static_cast<void> (0) : __assert_fail ("Entry != Inst->getParent() && \"PHI or landing pad in entry block!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 184, __PRETTY_FUNCTION__))
;
185 if (Idx != ~0U && isa<PHINode>(Inst))
186 return cast<PHINode>(Inst)->getIncomingBlock(Idx)->getTerminator();
187
188 // This must be an EH pad. Iterate over immediate dominators until we find a
189 // non-EH pad. We need to skip over catchswitch blocks, which are both EH pads
190 // and terminators.
191 auto IDom = DT->getNode(Inst->getParent())->getIDom();
192 while (IDom->getBlock()->isEHPad()) {
193 assert(Entry != IDom->getBlock() && "eh pad in entry block")((Entry != IDom->getBlock() && "eh pad in entry block"
) ? static_cast<void> (0) : __assert_fail ("Entry != IDom->getBlock() && \"eh pad in entry block\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 193, __PRETTY_FUNCTION__))
;
194 IDom = IDom->getIDom();
195 }
196
197 return IDom->getBlock()->getTerminator();
198}
199
200/// Given \p BBs as input, find another set of BBs which collectively
201/// dominates \p BBs and have the minimal sum of frequencies. Return the BB
202/// set found in \p BBs.
203static void findBestInsertionSet(DominatorTree &DT, BlockFrequencyInfo &BFI,
204 BasicBlock *Entry,
205 SmallPtrSet<BasicBlock *, 8> &BBs) {
206 assert(!BBs.count(Entry) && "Assume Entry is not in BBs")((!BBs.count(Entry) && "Assume Entry is not in BBs") ?
static_cast<void> (0) : __assert_fail ("!BBs.count(Entry) && \"Assume Entry is not in BBs\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 206, __PRETTY_FUNCTION__))
;
207 // Nodes on the current path to the root.
208 SmallPtrSet<BasicBlock *, 8> Path;
209 // Candidates includes any block 'BB' in set 'BBs' that is not strictly
210 // dominated by any other blocks in set 'BBs', and all nodes in the path
211 // in the dominator tree from Entry to 'BB'.
212 SmallPtrSet<BasicBlock *, 16> Candidates;
213 for (auto BB : BBs) {
214 Path.clear();
215 // Walk up the dominator tree until Entry or another BB in BBs
216 // is reached. Insert the nodes on the way to the Path.
217 BasicBlock *Node = BB;
218 // The "Path" is a candidate path to be added into Candidates set.
219 bool isCandidate = false;
220 do {
221 Path.insert(Node);
222 if (Node == Entry || Candidates.count(Node)) {
223 isCandidate = true;
224 break;
225 }
226 assert(DT.getNode(Node)->getIDom() &&((DT.getNode(Node)->getIDom() && "Entry doens't dominate current Node"
) ? static_cast<void> (0) : __assert_fail ("DT.getNode(Node)->getIDom() && \"Entry doens't dominate current Node\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 227, __PRETTY_FUNCTION__))
227 "Entry doens't dominate current Node")((DT.getNode(Node)->getIDom() && "Entry doens't dominate current Node"
) ? static_cast<void> (0) : __assert_fail ("DT.getNode(Node)->getIDom() && \"Entry doens't dominate current Node\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 227, __PRETTY_FUNCTION__))
;
228 Node = DT.getNode(Node)->getIDom()->getBlock();
229 } while (!BBs.count(Node));
230
231 // If isCandidate is false, Node is another Block in BBs dominating
232 // current 'BB'. Drop the nodes on the Path.
233 if (!isCandidate)
234 continue;
235
236 // Add nodes on the Path into Candidates.
237 Candidates.insert(Path.begin(), Path.end());
238 }
239
240 // Sort the nodes in Candidates in top-down order and save the nodes
241 // in Orders.
242 unsigned Idx = 0;
243 SmallVector<BasicBlock *, 16> Orders;
244 Orders.push_back(Entry);
245 while (Idx != Orders.size()) {
246 BasicBlock *Node = Orders[Idx++];
247 for (auto ChildDomNode : DT.getNode(Node)->getChildren()) {
248 if (Candidates.count(ChildDomNode->getBlock()))
249 Orders.push_back(ChildDomNode->getBlock());
250 }
251 }
252
253 // Visit Orders in bottom-up order.
254 using InsertPtsCostPair =
255 std::pair<SmallPtrSet<BasicBlock *, 16>, BlockFrequency>;
256
257 // InsertPtsMap is a map from a BB to the best insertion points for the
258 // subtree of BB (subtree not including the BB itself).
259 DenseMap<BasicBlock *, InsertPtsCostPair> InsertPtsMap;
260 InsertPtsMap.reserve(Orders.size() + 1);
261 for (auto RIt = Orders.rbegin(); RIt != Orders.rend(); RIt++) {
262 BasicBlock *Node = *RIt;
263 bool NodeInBBs = BBs.count(Node);
264 SmallPtrSet<BasicBlock *, 16> &InsertPts = InsertPtsMap[Node].first;
265 BlockFrequency &InsertPtsFreq = InsertPtsMap[Node].second;
266
267 // Return the optimal insert points in BBs.
268 if (Node == Entry) {
269 BBs.clear();
270 if (InsertPtsFreq > BFI.getBlockFreq(Node) ||
271 (InsertPtsFreq == BFI.getBlockFreq(Node) && InsertPts.size() > 1))
272 BBs.insert(Entry);
273 else
274 BBs.insert(InsertPts.begin(), InsertPts.end());
275 break;
276 }
277
278 BasicBlock *Parent = DT.getNode(Node)->getIDom()->getBlock();
279 // Initially, ParentInsertPts is empty and ParentPtsFreq is 0. Every child
280 // will update its parent's ParentInsertPts and ParentPtsFreq.
281 SmallPtrSet<BasicBlock *, 16> &ParentInsertPts = InsertPtsMap[Parent].first;
282 BlockFrequency &ParentPtsFreq = InsertPtsMap[Parent].second;
283 // Choose to insert in Node or in subtree of Node.
284 // Don't hoist to EHPad because we may not find a proper place to insert
285 // in EHPad.
286 // If the total frequency of InsertPts is the same as the frequency of the
287 // target Node, and InsertPts contains more than one nodes, choose hoisting
288 // to reduce code size.
289 if (NodeInBBs ||
290 (!Node->isEHPad() &&
291 (InsertPtsFreq > BFI.getBlockFreq(Node) ||
292 (InsertPtsFreq == BFI.getBlockFreq(Node) && InsertPts.size() > 1)))) {
293 ParentInsertPts.insert(Node);
294 ParentPtsFreq += BFI.getBlockFreq(Node);
295 } else {
296 ParentInsertPts.insert(InsertPts.begin(), InsertPts.end());
297 ParentPtsFreq += InsertPtsFreq;
298 }
299 }
300}
301
302/// Find an insertion point that dominates all uses.
303SmallPtrSet<Instruction *, 8> ConstantHoistingPass::findConstantInsertionPoint(
304 const ConstantInfo &ConstInfo) const {
305 assert(!ConstInfo.RebasedConstants.empty() && "Invalid constant info entry.")((!ConstInfo.RebasedConstants.empty() && "Invalid constant info entry."
) ? static_cast<void> (0) : __assert_fail ("!ConstInfo.RebasedConstants.empty() && \"Invalid constant info entry.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 305, __PRETTY_FUNCTION__))
;
12
'?' condition is true
306 // Collect all basic blocks.
307 SmallPtrSet<BasicBlock *, 8> BBs;
308 SmallPtrSet<Instruction *, 8> InsertPts;
309 for (auto const &RCI : ConstInfo.RebasedConstants)
13
Assuming '__begin1' is equal to '__end1'
310 for (auto const &U : RCI.Uses)
311 BBs.insert(findMatInsertPt(U.Inst, U.OpndIdx)->getParent());
312
313 if (BBs.count(Entry)) {
14
Assuming the condition is false
15
Taking false branch
314 InsertPts.insert(&Entry->front());
315 return InsertPts;
316 }
317
318 if (BFI) {
16
Assuming the condition is false
17
Taking false branch
319 findBestInsertionSet(*DT, *BFI, Entry, BBs);
320 for (auto BB : BBs) {
321 BasicBlock::iterator InsertPt = BB->begin();
322 for (; isa<PHINode>(InsertPt) || InsertPt->isEHPad(); ++InsertPt)
323 ;
324 InsertPts.insert(&*InsertPt);
325 }
326 return InsertPts;
327 }
328
329 while (BBs.size() >= 2) {
18
Assuming the condition is true
19
Loop condition is true. Entering loop body
330 BasicBlock *BB, *BB1, *BB2;
331 BB1 = *BBs.begin();
332 BB2 = *std::next(BBs.begin());
333 BB = DT->findNearestCommonDominator(BB1, BB2);
334 if (BB == Entry) {
20
Assuming pointer value is null
21
Taking true branch
335 InsertPts.insert(&Entry->front());
22
Called C++ object pointer is null
336 return InsertPts;
337 }
338 BBs.erase(BB1);
339 BBs.erase(BB2);
340 BBs.insert(BB);
341 }
342 assert((BBs.size() == 1) && "Expected only one element.")(((BBs.size() == 1) && "Expected only one element.") ?
static_cast<void> (0) : __assert_fail ("(BBs.size() == 1) && \"Expected only one element.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 342, __PRETTY_FUNCTION__))
;
343 Instruction &FirstInst = (*BBs.begin())->front();
344 InsertPts.insert(findMatInsertPt(&FirstInst));
345 return InsertPts;
346}
347
348/// Record constant integer ConstInt for instruction Inst at operand
349/// index Idx.
350///
351/// The operand at index Idx is not necessarily the constant integer itself. It
352/// could also be a cast instruction or a constant expression that uses the
353/// constant integer.
354void ConstantHoistingPass::collectConstantCandidates(
355 ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx,
356 ConstantInt *ConstInt) {
357 unsigned Cost;
358 // Ask the target about the cost of materializing the constant for the given
359 // instruction and operand index.
360 if (auto IntrInst = dyn_cast<IntrinsicInst>(Inst))
361 Cost = TTI->getIntImmCost(IntrInst->getIntrinsicID(), Idx,
362 ConstInt->getValue(), ConstInt->getType());
363 else
364 Cost = TTI->getIntImmCost(Inst->getOpcode(), Idx, ConstInt->getValue(),
365 ConstInt->getType());
366
367 // Ignore cheap integer constants.
368 if (Cost > TargetTransformInfo::TCC_Basic) {
369 ConstCandMapType::iterator Itr;
370 bool Inserted;
371 ConstPtrUnionType Cand = ConstInt;
372 std::tie(Itr, Inserted) = ConstCandMap.insert(std::make_pair(Cand, 0));
373 if (Inserted) {
374 ConstIntCandVec.push_back(ConstantCandidate(ConstInt));
375 Itr->second = ConstIntCandVec.size() - 1;
376 }
377 ConstIntCandVec[Itr->second].addUser(Inst, Idx, Cost);
378 LLVM_DEBUG(if (isa<ConstantInt>(Inst->getOperand(Idx))) dbgs()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { if (isa<ConstantInt>(Inst->getOperand
(Idx))) dbgs() << "Collect constant " << *ConstInt
<< " from " << *Inst << " with cost " <<
Cost << '\n'; else dbgs() << "Collect constant "
<< *ConstInt << " indirectly from " << *Inst
<< " via " << *Inst->getOperand(Idx) <<
" with cost " << Cost << '\n';; } } while (false
)
379 << "Collect constant " << *ConstInt << " from " << *Instdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { if (isa<ConstantInt>(Inst->getOperand
(Idx))) dbgs() << "Collect constant " << *ConstInt
<< " from " << *Inst << " with cost " <<
Cost << '\n'; else dbgs() << "Collect constant "
<< *ConstInt << " indirectly from " << *Inst
<< " via " << *Inst->getOperand(Idx) <<
" with cost " << Cost << '\n';; } } while (false
)
380 << " with cost " << Cost << '\n';do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { if (isa<ConstantInt>(Inst->getOperand
(Idx))) dbgs() << "Collect constant " << *ConstInt
<< " from " << *Inst << " with cost " <<
Cost << '\n'; else dbgs() << "Collect constant "
<< *ConstInt << " indirectly from " << *Inst
<< " via " << *Inst->getOperand(Idx) <<
" with cost " << Cost << '\n';; } } while (false
)
381 else dbgs() << "Collect constant " << *ConstIntdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { if (isa<ConstantInt>(Inst->getOperand
(Idx))) dbgs() << "Collect constant " << *ConstInt
<< " from " << *Inst << " with cost " <<
Cost << '\n'; else dbgs() << "Collect constant "
<< *ConstInt << " indirectly from " << *Inst
<< " via " << *Inst->getOperand(Idx) <<
" with cost " << Cost << '\n';; } } while (false
)
382 << " indirectly from " << *Inst << " via "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { if (isa<ConstantInt>(Inst->getOperand
(Idx))) dbgs() << "Collect constant " << *ConstInt
<< " from " << *Inst << " with cost " <<
Cost << '\n'; else dbgs() << "Collect constant "
<< *ConstInt << " indirectly from " << *Inst
<< " via " << *Inst->getOperand(Idx) <<
" with cost " << Cost << '\n';; } } while (false
)
383 << *Inst->getOperand(Idx) << " with cost " << Costdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { if (isa<ConstantInt>(Inst->getOperand
(Idx))) dbgs() << "Collect constant " << *ConstInt
<< " from " << *Inst << " with cost " <<
Cost << '\n'; else dbgs() << "Collect constant "
<< *ConstInt << " indirectly from " << *Inst
<< " via " << *Inst->getOperand(Idx) <<
" with cost " << Cost << '\n';; } } while (false
)
384 << '\n';)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { if (isa<ConstantInt>(Inst->getOperand
(Idx))) dbgs() << "Collect constant " << *ConstInt
<< " from " << *Inst << " with cost " <<
Cost << '\n'; else dbgs() << "Collect constant "
<< *ConstInt << " indirectly from " << *Inst
<< " via " << *Inst->getOperand(Idx) <<
" with cost " << Cost << '\n';; } } while (false
)
;
385 }
386}
387
388/// Record constant GEP expression for instruction Inst at operand index Idx.
389void ConstantHoistingPass::collectConstantCandidates(
390 ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx,
391 ConstantExpr *ConstExpr) {
392 // TODO: Handle vector GEPs
393 if (ConstExpr->getType()->isVectorTy())
394 return;
395
396 GlobalVariable *BaseGV = dyn_cast<GlobalVariable>(ConstExpr->getOperand(0));
397 if (!BaseGV)
398 return;
399
400 // Get offset from the base GV.
401 PointerType *GVPtrTy = dyn_cast<PointerType>(BaseGV->getType());
402 IntegerType *PtrIntTy = DL->getIntPtrType(*Ctx, GVPtrTy->getAddressSpace());
403 APInt Offset(DL->getTypeSizeInBits(PtrIntTy), /*val*/0, /*isSigned*/true);
404 auto *GEPO = cast<GEPOperator>(ConstExpr);
405 if (!GEPO->accumulateConstantOffset(*DL, Offset))
406 return;
407
408 if (!Offset.isIntN(32))
409 return;
410
411 // A constant GEP expression that has a GlobalVariable as base pointer is
412 // usually lowered to a load from constant pool. Such operation is unlikely
413 // to be cheaper than compute it by <Base + Offset>, which can be lowered to
414 // an ADD instruction or folded into Load/Store instruction.
415 int Cost = TTI->getIntImmCost(Instruction::Add, 1, Offset, PtrIntTy);
416 ConstCandVecType &ExprCandVec = ConstGEPCandMap[BaseGV];
417 ConstCandMapType::iterator Itr;
418 bool Inserted;
419 ConstPtrUnionType Cand = ConstExpr;
420 std::tie(Itr, Inserted) = ConstCandMap.insert(std::make_pair(Cand, 0));
421 if (Inserted) {
422 ExprCandVec.push_back(ConstantCandidate(
423 ConstantInt::get(Type::getInt32Ty(*Ctx), Offset.getLimitedValue()),
424 ConstExpr));
425 Itr->second = ExprCandVec.size() - 1;
426 }
427 ExprCandVec[Itr->second].addUser(Inst, Idx, Cost);
428}
429
430/// Check the operand for instruction Inst at index Idx.
431void ConstantHoistingPass::collectConstantCandidates(
432 ConstCandMapType &ConstCandMap, Instruction *Inst, unsigned Idx) {
433 Value *Opnd = Inst->getOperand(Idx);
434
435 // Visit constant integers.
436 if (auto ConstInt = dyn_cast<ConstantInt>(Opnd)) {
437 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
438 return;
439 }
440
441 // Visit cast instructions that have constant integers.
442 if (auto CastInst = dyn_cast<Instruction>(Opnd)) {
443 // Only visit cast instructions, which have been skipped. All other
444 // instructions should have already been visited.
445 if (!CastInst->isCast())
446 return;
447
448 if (auto *ConstInt = dyn_cast<ConstantInt>(CastInst->getOperand(0))) {
449 // Pretend the constant is directly used by the instruction and ignore
450 // the cast instruction.
451 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
452 return;
453 }
454 }
455
456 // Visit constant expressions that have constant integers.
457 if (auto ConstExpr = dyn_cast<ConstantExpr>(Opnd)) {
458 // Handle constant gep expressions.
459 if (ConstHoistGEP && ConstExpr->isGEPWithNoNotionalOverIndexing())
460 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstExpr);
461
462 // Only visit constant cast expressions.
463 if (!ConstExpr->isCast())
464 return;
465
466 if (auto ConstInt = dyn_cast<ConstantInt>(ConstExpr->getOperand(0))) {
467 // Pretend the constant is directly used by the instruction and ignore
468 // the constant expression.
469 collectConstantCandidates(ConstCandMap, Inst, Idx, ConstInt);
470 return;
471 }
472 }
473}
474
475/// Scan the instruction for expensive integer constants and record them
476/// in the constant candidate vector.
477void ConstantHoistingPass::collectConstantCandidates(
478 ConstCandMapType &ConstCandMap, Instruction *Inst) {
479 // Skip all cast instructions. They are visited indirectly later on.
480 if (Inst->isCast())
481 return;
482
483 // Scan all operands.
484 for (unsigned Idx = 0, E = Inst->getNumOperands(); Idx != E; ++Idx) {
485 // The cost of materializing the constants (defined in
486 // `TargetTransformInfo::getIntImmCost`) for instructions which only take
487 // constant variables is lower than `TargetTransformInfo::TCC_Basic`. So
488 // it's safe for us to collect constant candidates from all IntrinsicInsts.
489 if (canReplaceOperandWithVariable(Inst, Idx) || isa<IntrinsicInst>(Inst)) {
490 collectConstantCandidates(ConstCandMap, Inst, Idx);
491 }
492 } // end of for all operands
493}
494
495/// Collect all integer constants in the function that cannot be folded
496/// into an instruction itself.
497void ConstantHoistingPass::collectConstantCandidates(Function &Fn) {
498 ConstCandMapType ConstCandMap;
499 for (BasicBlock &BB : Fn)
500 for (Instruction &Inst : BB)
501 collectConstantCandidates(ConstCandMap, &Inst);
502}
503
504// This helper function is necessary to deal with values that have different
505// bit widths (APInt Operator- does not like that). If the value cannot be
506// represented in uint64 we return an "empty" APInt. This is then interpreted
507// as the value is not in range.
508static Optional<APInt> calculateOffsetDiff(const APInt &V1, const APInt &V2) {
509 Optional<APInt> Res = None;
510 unsigned BW = V1.getBitWidth() > V2.getBitWidth() ?
511 V1.getBitWidth() : V2.getBitWidth();
512 uint64_t LimVal1 = V1.getLimitedValue();
513 uint64_t LimVal2 = V2.getLimitedValue();
514
515 if (LimVal1 == ~0ULL || LimVal2 == ~0ULL)
516 return Res;
517
518 uint64_t Diff = LimVal1 - LimVal2;
519 return APInt(BW, Diff, true);
520}
521
522// From a list of constants, one needs to picked as the base and the other
523// constants will be transformed into an offset from that base constant. The
524// question is which we can pick best? For example, consider these constants
525// and their number of uses:
526//
527// Constants| 2 | 4 | 12 | 42 |
528// NumUses | 3 | 2 | 8 | 7 |
529//
530// Selecting constant 12 because it has the most uses will generate negative
531// offsets for constants 2 and 4 (i.e. -10 and -8 respectively). If negative
532// offsets lead to less optimal code generation, then there might be better
533// solutions. Suppose immediates in the range of 0..35 are most optimally
534// supported by the architecture, then selecting constant 2 is most optimal
535// because this will generate offsets: 0, 2, 10, 40. Offsets 0, 2 and 10 are in
536// range 0..35, and thus 3 + 2 + 8 = 13 uses are in range. Selecting 12 would
537// have only 8 uses in range, so choosing 2 as a base is more optimal. Thus, in
538// selecting the base constant the range of the offsets is a very important
539// factor too that we take into account here. This algorithm calculates a total
540// costs for selecting a constant as the base and substract the costs if
541// immediates are out of range. It has quadratic complexity, so we call this
542// function only when we're optimising for size and there are less than 100
543// constants, we fall back to the straightforward algorithm otherwise
544// which does not do all the offset calculations.
545unsigned
546ConstantHoistingPass::maximizeConstantsInRange(ConstCandVecType::iterator S,
547 ConstCandVecType::iterator E,
548 ConstCandVecType::iterator &MaxCostItr) {
549 unsigned NumUses = 0;
550
551 if(!Entry->getParent()->optForSize() || std::distance(S,E) > 100) {
552 for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
553 NumUses += ConstCand->Uses.size();
554 if (ConstCand->CumulativeCost > MaxCostItr->CumulativeCost)
555 MaxCostItr = ConstCand;
556 }
557 return NumUses;
558 }
559
560 LLVM_DEBUG(dbgs() << "== Maximize constants in range ==\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "== Maximize constants in range ==\n"
; } } while (false)
;
561 int MaxCost = -1;
562 for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
563 auto Value = ConstCand->ConstInt->getValue();
564 Type *Ty = ConstCand->ConstInt->getType();
565 int Cost = 0;
566 NumUses += ConstCand->Uses.size();
567 LLVM_DEBUG(dbgs() << "= Constant: " << ConstCand->ConstInt->getValue()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "= Constant: " << ConstCand
->ConstInt->getValue() << "\n"; } } while (false)
568 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "= Constant: " << ConstCand
->ConstInt->getValue() << "\n"; } } while (false)
;
569
570 for (auto User : ConstCand->Uses) {
571 unsigned Opcode = User.Inst->getOpcode();
572 unsigned OpndIdx = User.OpndIdx;
573 Cost += TTI->getIntImmCost(Opcode, OpndIdx, Value, Ty);
574 LLVM_DEBUG(dbgs() << "Cost: " << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Cost: " << Cost <<
"\n"; } } while (false)
;
575
576 for (auto C2 = S; C2 != E; ++C2) {
577 Optional<APInt> Diff = calculateOffsetDiff(
578 C2->ConstInt->getValue(),
579 ConstCand->ConstInt->getValue());
580 if (Diff) {
581 const int ImmCosts =
582 TTI->getIntImmCodeSizeCost(Opcode, OpndIdx, Diff.getValue(), Ty);
583 Cost -= ImmCosts;
584 LLVM_DEBUG(dbgs() << "Offset " << Diff.getValue() << " "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Offset " << Diff.getValue
() << " " << "has penalty: " << ImmCosts <<
"\n" << "Adjusted cost: " << Cost << "\n";
} } while (false)
585 << "has penalty: " << ImmCosts << "\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Offset " << Diff.getValue
() << " " << "has penalty: " << ImmCosts <<
"\n" << "Adjusted cost: " << Cost << "\n";
} } while (false)
586 << "Adjusted cost: " << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Offset " << Diff.getValue
() << " " << "has penalty: " << ImmCosts <<
"\n" << "Adjusted cost: " << Cost << "\n";
} } while (false)
;
587 }
588 }
589 }
590 LLVM_DEBUG(dbgs() << "Cumulative cost: " << Cost << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Cumulative cost: " <<
Cost << "\n"; } } while (false)
;
591 if (Cost > MaxCost) {
592 MaxCost = Cost;
593 MaxCostItr = ConstCand;
594 LLVM_DEBUG(dbgs() << "New candidate: " << MaxCostItr->ConstInt->getValue()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "New candidate: " << MaxCostItr
->ConstInt->getValue() << "\n"; } } while (false)
595 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "New candidate: " << MaxCostItr
->ConstInt->getValue() << "\n"; } } while (false)
;
596 }
597 }
598 return NumUses;
599}
600
601/// Find the base constant within the given range and rebase all other
602/// constants with respect to the base constant.
603void ConstantHoistingPass::findAndMakeBaseConstant(
604 ConstCandVecType::iterator S, ConstCandVecType::iterator E,
605 SmallVectorImpl<consthoist::ConstantInfo> &ConstInfoVec) {
606 auto MaxCostItr = S;
607 unsigned NumUses = maximizeConstantsInRange(S, E, MaxCostItr);
608
609 // Don't hoist constants that have only one use.
610 if (NumUses <= 1)
611 return;
612
613 ConstantInt *ConstInt = MaxCostItr->ConstInt;
614 ConstantExpr *ConstExpr = MaxCostItr->ConstExpr;
615 ConstantInfo ConstInfo;
616 ConstInfo.BaseInt = ConstInt;
617 ConstInfo.BaseExpr = ConstExpr;
618 Type *Ty = ConstInt->getType();
619
620 // Rebase the constants with respect to the base constant.
621 for (auto ConstCand = S; ConstCand != E; ++ConstCand) {
622 APInt Diff = ConstCand->ConstInt->getValue() - ConstInt->getValue();
623 Constant *Offset = Diff == 0 ? nullptr : ConstantInt::get(Ty, Diff);
624 Type *ConstTy =
625 ConstCand->ConstExpr ? ConstCand->ConstExpr->getType() : nullptr;
626 ConstInfo.RebasedConstants.push_back(
627 RebasedConstantInfo(std::move(ConstCand->Uses), Offset, ConstTy));
628 }
629 ConstInfoVec.push_back(std::move(ConstInfo));
630}
631
632/// Finds and combines constant candidates that can be easily
633/// rematerialized with an add from a common base constant.
634void ConstantHoistingPass::findBaseConstants(GlobalVariable *BaseGV) {
635 // If BaseGV is nullptr, find base among candidate constant integers;
636 // Otherwise find base among constant GEPs that share the same BaseGV.
637 ConstCandVecType &ConstCandVec = BaseGV ?
638 ConstGEPCandMap[BaseGV] : ConstIntCandVec;
639 ConstInfoVecType &ConstInfoVec = BaseGV ?
640 ConstGEPInfoMap[BaseGV] : ConstIntInfoVec;
641
642 // Sort the constants by value and type. This invalidates the mapping!
643 std::stable_sort(ConstCandVec.begin(), ConstCandVec.end(),
644 [](const ConstantCandidate &LHS, const ConstantCandidate &RHS) {
645 if (LHS.ConstInt->getType() != RHS.ConstInt->getType())
646 return LHS.ConstInt->getType()->getBitWidth() <
647 RHS.ConstInt->getType()->getBitWidth();
648 return LHS.ConstInt->getValue().ult(RHS.ConstInt->getValue());
649 });
650
651 // Simple linear scan through the sorted constant candidate vector for viable
652 // merge candidates.
653 auto MinValItr = ConstCandVec.begin();
654 for (auto CC = std::next(ConstCandVec.begin()), E = ConstCandVec.end();
655 CC != E; ++CC) {
656 if (MinValItr->ConstInt->getType() == CC->ConstInt->getType()) {
657 Type *MemUseValTy = nullptr;
658 for (auto &U : CC->Uses) {
659 auto *UI = U.Inst;
660 if (LoadInst *LI = dyn_cast<LoadInst>(UI)) {
661 MemUseValTy = LI->getType();
662 break;
663 } else if (StoreInst *SI = dyn_cast<StoreInst>(UI)) {
664 // Make sure the constant is used as pointer operand of the StoreInst.
665 if (SI->getPointerOperand() == SI->getOperand(U.OpndIdx)) {
666 MemUseValTy = SI->getValueOperand()->getType();
667 break;
668 }
669 }
670 }
671
672 // Check if the constant is in range of an add with immediate.
673 APInt Diff = CC->ConstInt->getValue() - MinValItr->ConstInt->getValue();
674 if ((Diff.getBitWidth() <= 64) &&
675 TTI->isLegalAddImmediate(Diff.getSExtValue()) &&
676 // Check if Diff can be used as offset in addressing mode of the user
677 // memory instruction.
678 (!MemUseValTy || TTI->isLegalAddressingMode(MemUseValTy,
679 /*BaseGV*/nullptr, /*BaseOffset*/Diff.getSExtValue(),
680 /*HasBaseReg*/true, /*Scale*/0)))
681 continue;
682 }
683 // We either have now a different constant type or the constant is not in
684 // range of an add with immediate anymore.
685 findAndMakeBaseConstant(MinValItr, CC, ConstInfoVec);
686 // Start a new base constant search.
687 MinValItr = CC;
688 }
689 // Finalize the last base constant search.
690 findAndMakeBaseConstant(MinValItr, ConstCandVec.end(), ConstInfoVec);
691}
692
693/// Updates the operand at Idx in instruction Inst with the result of
694/// instruction Mat. If the instruction is a PHI node then special
695/// handling for duplicate values form the same incoming basic block is
696/// required.
697/// \return The update will always succeed, but the return value indicated if
698/// Mat was used for the update or not.
699static bool updateOperand(Instruction *Inst, unsigned Idx, Instruction *Mat) {
700 if (auto PHI = dyn_cast<PHINode>(Inst)) {
701 // Check if any previous operand of the PHI node has the same incoming basic
702 // block. This is a very odd case that happens when the incoming basic block
703 // has a switch statement. In this case use the same value as the previous
704 // operand(s), otherwise we will fail verification due to different values.
705 // The values are actually the same, but the variable names are different
706 // and the verifier doesn't like that.
707 BasicBlock *IncomingBB = PHI->getIncomingBlock(Idx);
708 for (unsigned i = 0; i < Idx; ++i) {
709 if (PHI->getIncomingBlock(i) == IncomingBB) {
710 Value *IncomingVal = PHI->getIncomingValue(i);
711 Inst->setOperand(Idx, IncomingVal);
712 return false;
713 }
714 }
715 }
716
717 Inst->setOperand(Idx, Mat);
718 return true;
719}
720
721/// Emit materialization code for all rebased constants and update their
722/// users.
723void ConstantHoistingPass::emitBaseConstants(Instruction *Base,
724 Constant *Offset,
725 Type *Ty,
726 const ConstantUser &ConstUser) {
727 Instruction *Mat = Base;
728
729 // The same offset can be dereferenced to different types in nested struct.
730 if (!Offset && Ty && Ty != Base->getType())
731 Offset = ConstantInt::get(Type::getInt32Ty(*Ctx), 0);
732
733 if (Offset) {
734 Instruction *InsertionPt = findMatInsertPt(ConstUser.Inst,
735 ConstUser.OpndIdx);
736 if (Ty) {
737 // Constant being rebased is a ConstantExpr.
738 PointerType *Int8PtrTy = Type::getInt8PtrTy(*Ctx,
739 cast<PointerType>(Ty)->getAddressSpace());
740 Base = new BitCastInst(Base, Int8PtrTy, "base_bitcast", InsertionPt);
741 Mat = GetElementPtrInst::Create(Int8PtrTy->getElementType(), Base,
742 Offset, "mat_gep", InsertionPt);
743 Mat = new BitCastInst(Mat, Ty, "mat_bitcast", InsertionPt);
744 } else
745 // Constant being rebased is a ConstantInt.
746 Mat = BinaryOperator::Create(Instruction::Add, Base, Offset,
747 "const_mat", InsertionPt);
748
749 LLVM_DEBUG(dbgs() << "Materialize constant (" << *Base->getOperand(0)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Materialize constant (" <<
*Base->getOperand(0) << " + " << *Offset <<
") in BB " << Mat->getParent()->getName() <<
'\n' << *Mat << '\n'; } } while (false)
750 << " + " << *Offset << ") in BB "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Materialize constant (" <<
*Base->getOperand(0) << " + " << *Offset <<
") in BB " << Mat->getParent()->getName() <<
'\n' << *Mat << '\n'; } } while (false)
751 << Mat->getParent()->getName() << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Materialize constant (" <<
*Base->getOperand(0) << " + " << *Offset <<
") in BB " << Mat->getParent()->getName() <<
'\n' << *Mat << '\n'; } } while (false)
752 << *Mat << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Materialize constant (" <<
*Base->getOperand(0) << " + " << *Offset <<
") in BB " << Mat->getParent()->getName() <<
'\n' << *Mat << '\n'; } } while (false)
;
753 Mat->setDebugLoc(ConstUser.Inst->getDebugLoc());
754 }
755 Value *Opnd = ConstUser.Inst->getOperand(ConstUser.OpndIdx);
756
757 // Visit constant integer.
758 if (isa<ConstantInt>(Opnd)) {
759 LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Update: " << *ConstUser
.Inst << '\n'; } } while (false)
;
760 if (!updateOperand(ConstUser.Inst, ConstUser.OpndIdx, Mat) && Offset)
761 Mat->eraseFromParent();
762 LLVM_DEBUG(dbgs() << "To : " << *ConstUser.Inst << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "To : " << *ConstUser
.Inst << '\n'; } } while (false)
;
763 return;
764 }
765
766 // Visit cast instruction.
767 if (auto CastInst = dyn_cast<Instruction>(Opnd)) {
768 assert(CastInst->isCast() && "Expected an cast instruction!")((CastInst->isCast() && "Expected an cast instruction!"
) ? static_cast<void> (0) : __assert_fail ("CastInst->isCast() && \"Expected an cast instruction!\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 768, __PRETTY_FUNCTION__))
;
769 // Check if we already have visited this cast instruction before to avoid
770 // unnecessary cloning.
771 Instruction *&ClonedCastInst = ClonedCastMap[CastInst];
772 if (!ClonedCastInst) {
773 ClonedCastInst = CastInst->clone();
774 ClonedCastInst->setOperand(0, Mat);
775 ClonedCastInst->insertAfter(CastInst);
776 // Use the same debug location as the original cast instruction.
777 ClonedCastInst->setDebugLoc(CastInst->getDebugLoc());
778 LLVM_DEBUG(dbgs() << "Clone instruction: " << *CastInst << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Clone instruction: " <<
*CastInst << '\n' << "To : " <<
*ClonedCastInst << '\n'; } } while (false)
779 << "To : " << *ClonedCastInst << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Clone instruction: " <<
*CastInst << '\n' << "To : " <<
*ClonedCastInst << '\n'; } } while (false)
;
780 }
781
782 LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Update: " << *ConstUser
.Inst << '\n'; } } while (false)
;
783 updateOperand(ConstUser.Inst, ConstUser.OpndIdx, ClonedCastInst);
784 LLVM_DEBUG(dbgs() << "To : " << *ConstUser.Inst << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "To : " << *ConstUser
.Inst << '\n'; } } while (false)
;
785 return;
786 }
787
788 // Visit constant expression.
789 if (auto ConstExpr = dyn_cast<ConstantExpr>(Opnd)) {
790 if (ConstExpr->isGEPWithNoNotionalOverIndexing()) {
791 // Operand is a ConstantGEP, replace it.
792 updateOperand(ConstUser.Inst, ConstUser.OpndIdx, Mat);
793 return;
794 }
795
796 // Aside from constant GEPs, only constant cast expressions are collected.
797 assert(ConstExpr->isCast() && "ConstExpr should be a cast")((ConstExpr->isCast() && "ConstExpr should be a cast"
) ? static_cast<void> (0) : __assert_fail ("ConstExpr->isCast() && \"ConstExpr should be a cast\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 797, __PRETTY_FUNCTION__))
;
798 Instruction *ConstExprInst = ConstExpr->getAsInstruction();
799 ConstExprInst->setOperand(0, Mat);
800 ConstExprInst->insertBefore(findMatInsertPt(ConstUser.Inst,
801 ConstUser.OpndIdx));
802
803 // Use the same debug location as the instruction we are about to update.
804 ConstExprInst->setDebugLoc(ConstUser.Inst->getDebugLoc());
805
806 LLVM_DEBUG(dbgs() << "Create instruction: " << *ConstExprInst << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Create instruction: " <<
*ConstExprInst << '\n' << "From : "
<< *ConstExpr << '\n'; } } while (false)
807 << "From : " << *ConstExpr << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Create instruction: " <<
*ConstExprInst << '\n' << "From : "
<< *ConstExpr << '\n'; } } while (false)
;
808 LLVM_DEBUG(dbgs() << "Update: " << *ConstUser.Inst << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Update: " << *ConstUser
.Inst << '\n'; } } while (false)
;
809 if (!updateOperand(ConstUser.Inst, ConstUser.OpndIdx, ConstExprInst)) {
810 ConstExprInst->eraseFromParent();
811 if (Offset)
812 Mat->eraseFromParent();
813 }
814 LLVM_DEBUG(dbgs() << "To : " << *ConstUser.Inst << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "To : " << *ConstUser
.Inst << '\n'; } } while (false)
;
815 return;
816 }
817}
818
819/// Hoist and hide the base constant behind a bitcast and emit
820/// materialization code for derived constants.
821bool ConstantHoistingPass::emitBaseConstants(GlobalVariable *BaseGV) {
822 bool MadeChange = false;
823 SmallVectorImpl<consthoist::ConstantInfo> &ConstInfoVec =
824 BaseGV ? ConstGEPInfoMap[BaseGV] : ConstIntInfoVec;
8
'?' condition is false
825 for (auto const &ConstInfo : ConstInfoVec) {
9
Value assigned to field 'Entry'
10
Assuming '__begin1' is not equal to '__end1'
826 SmallPtrSet<Instruction *, 8> IPSet = findConstantInsertionPoint(ConstInfo);
11
Calling 'ConstantHoistingPass::findConstantInsertionPoint'
827 assert(!IPSet.empty() && "IPSet is empty")((!IPSet.empty() && "IPSet is empty") ? static_cast<
void> (0) : __assert_fail ("!IPSet.empty() && \"IPSet is empty\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 827, __PRETTY_FUNCTION__))
;
828
829 unsigned UsesNum = 0;
830 unsigned ReBasesNum = 0;
831 unsigned NotRebasedNum = 0;
832 for (Instruction *IP : IPSet) {
833 // First, collect constants depending on this IP of the base.
834 unsigned Uses = 0;
835 using RebasedUse = std::tuple<Constant *, Type *, ConstantUser>;
836 SmallVector<RebasedUse, 4> ToBeRebased;
837 for (auto const &RCI : ConstInfo.RebasedConstants) {
838 for (auto const &U : RCI.Uses) {
839 Uses++;
840 BasicBlock *OrigMatInsertBB =
841 findMatInsertPt(U.Inst, U.OpndIdx)->getParent();
842 // If Base constant is to be inserted in multiple places,
843 // generate rebase for U using the Base dominating U.
844 if (IPSet.size() == 1 ||
845 DT->dominates(IP->getParent(), OrigMatInsertBB))
846 ToBeRebased.push_back(RebasedUse(RCI.Offset, RCI.Ty, U));
847 }
848 }
849 UsesNum = Uses;
850
851 // If only few constants depend on this IP of base, skip rebasing,
852 // assuming the base and the rebased have the same materialization cost.
853 if (ToBeRebased.size() < MinNumOfDependentToRebase) {
854 NotRebasedNum += ToBeRebased.size();
855 continue;
856 }
857
858 // Emit an instance of the base at this IP.
859 Instruction *Base = nullptr;
860 // Hoist and hide the base constant behind a bitcast.
861 if (ConstInfo.BaseExpr) {
862 assert(BaseGV && "A base constant expression must have an base GV")((BaseGV && "A base constant expression must have an base GV"
) ? static_cast<void> (0) : __assert_fail ("BaseGV && \"A base constant expression must have an base GV\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 862, __PRETTY_FUNCTION__))
;
863 Type *Ty = ConstInfo.BaseExpr->getType();
864 Base = new BitCastInst(ConstInfo.BaseExpr, Ty, "const", IP);
865 } else {
866 IntegerType *Ty = ConstInfo.BaseInt->getType();
867 Base = new BitCastInst(ConstInfo.BaseInt, Ty, "const", IP);
868 }
869
870 Base->setDebugLoc(IP->getDebugLoc());
871
872 LLVM_DEBUG(dbgs() << "Hoist constant (" << *ConstInfo.BaseIntdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Hoist constant (" <<
*ConstInfo.BaseInt << ") to BB " << IP->getParent
()->getName() << '\n' << *Base << '\n'; }
} while (false)
873 << ") to BB " << IP->getParent()->getName() << '\n'do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Hoist constant (" <<
*ConstInfo.BaseInt << ") to BB " << IP->getParent
()->getName() << '\n' << *Base << '\n'; }
} while (false)
874 << *Base << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("consthoist")) { dbgs() << "Hoist constant (" <<
*ConstInfo.BaseInt << ") to BB " << IP->getParent
()->getName() << '\n' << *Base << '\n'; }
} while (false)
;
875
876 // Emit materialization code for rebased constants depending on this IP.
877 for (auto const &R : ToBeRebased) {
878 Constant *Off = std::get<0>(R);
879 Type *Ty = std::get<1>(R);
880 ConstantUser U = std::get<2>(R);
881 emitBaseConstants(Base, Off, Ty, U);
882 ReBasesNum++;
883 // Use the same debug location as the last user of the constant.
884 Base->setDebugLoc(DILocation::getMergedLocation(
885 Base->getDebugLoc(), U.Inst->getDebugLoc()));
886 }
887 assert(!Base->use_empty() && "The use list is empty!?")((!Base->use_empty() && "The use list is empty!?")
? static_cast<void> (0) : __assert_fail ("!Base->use_empty() && \"The use list is empty!?\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 887, __PRETTY_FUNCTION__))
;
888 assert(isa<Instruction>(Base->user_back()) &&((isa<Instruction>(Base->user_back()) && "All uses should be instructions."
) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(Base->user_back()) && \"All uses should be instructions.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 889, __PRETTY_FUNCTION__))
889 "All uses should be instructions.")((isa<Instruction>(Base->user_back()) && "All uses should be instructions."
) ? static_cast<void> (0) : __assert_fail ("isa<Instruction>(Base->user_back()) && \"All uses should be instructions.\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 889, __PRETTY_FUNCTION__))
;
890 }
891 (void)UsesNum;
892 (void)ReBasesNum;
893 (void)NotRebasedNum;
894 // Expect all uses are rebased after rebase is done.
895 assert(UsesNum == (ReBasesNum + NotRebasedNum) &&((UsesNum == (ReBasesNum + NotRebasedNum) && "Not all uses are rebased"
) ? static_cast<void> (0) : __assert_fail ("UsesNum == (ReBasesNum + NotRebasedNum) && \"Not all uses are rebased\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 896, __PRETTY_FUNCTION__))
896 "Not all uses are rebased")((UsesNum == (ReBasesNum + NotRebasedNum) && "Not all uses are rebased"
) ? static_cast<void> (0) : __assert_fail ("UsesNum == (ReBasesNum + NotRebasedNum) && \"Not all uses are rebased\""
, "/build/llvm-toolchain-snapshot-8~svn350071/lib/Transforms/Scalar/ConstantHoisting.cpp"
, 896, __PRETTY_FUNCTION__))
;
897
898 NumConstantsHoisted++;
899
900 // Base constant is also included in ConstInfo.RebasedConstants, so
901 // deduct 1 from ConstInfo.RebasedConstants.size().
902 NumConstantsRebased += ConstInfo.RebasedConstants.size() - 1;
903
904 MadeChange = true;
905 }
906 return MadeChange;
907}
908
909/// Check all cast instructions we made a copy of and remove them if they
910/// have no more users.
911void ConstantHoistingPass::deleteDeadCastInst() const {
912 for (auto const &I : ClonedCastMap)
913 if (I.first->use_empty())
914 I.first->eraseFromParent();
915}
916
917/// Optimize expensive integer constants in the given function.
918bool ConstantHoistingPass::runImpl(Function &Fn, TargetTransformInfo &TTI,
919 DominatorTree &DT, BlockFrequencyInfo *BFI,
920 BasicBlock &Entry) {
921 this->TTI = &TTI;
922 this->DT = &DT;
923 this->BFI = BFI;
924 this->DL = &Fn.getParent()->getDataLayout();
925 this->Ctx = &Fn.getContext();
926 this->Entry = &Entry;
927 // Collect all constant candidates.
928 collectConstantCandidates(Fn);
929
930 // Combine constants that can be easily materialized with an add from a common
931 // base constant.
932 if (!ConstIntCandVec.empty())
4
Assuming the condition is false
5
Taking false branch
933 findBaseConstants(nullptr);
934 for (auto &MapEntry : ConstGEPCandMap)
935 if (!MapEntry.second.empty())
936 findBaseConstants(MapEntry.first);
937
938 // Finally hoist the base constant and emit materialization code for dependent
939 // constants.
940 bool MadeChange = false;
941 if (!ConstIntInfoVec.empty())
6
Taking true branch
942 MadeChange = emitBaseConstants(nullptr);
7
Calling 'ConstantHoistingPass::emitBaseConstants'
943 for (auto MapEntry : ConstGEPInfoMap)
944 if (!MapEntry.second.empty())
945 MadeChange |= emitBaseConstants(MapEntry.first);
946
947
948 // Cleanup dead instructions.
949 deleteDeadCastInst();
950
951 return MadeChange;
952}
953
954PreservedAnalyses ConstantHoistingPass::run(Function &F,
955 FunctionAnalysisManager &AM) {
956 auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
957 auto &TTI = AM.getResult<TargetIRAnalysis>(F);
958 auto BFI = ConstHoistWithBlockFrequency
1
Assuming the condition is false
2
'?' condition is false
959 ? &AM.getResult<BlockFrequencyAnalysis>(F)
960 : nullptr;
961 if (!runImpl(F, TTI, DT, BFI, F.getEntryBlock()))
3
Calling 'ConstantHoistingPass::runImpl'
962 return PreservedAnalyses::all();
963
964 PreservedAnalyses PA;
965 PA.preserveSet<CFGAnalyses>();
966 return PA;
967}