Bug Summary

File:include/llvm/Support/Error.h
Warning:line 201, column 5
Potential leak of memory pointed to by 'Payload._M_t._M_head_impl'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name PGOInstrumentation.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn345461/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/lib/Transforms/Instrumentation -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-10-27-211344-32123-1 -x c++ /build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp -faddrsig

/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp

1//===- PGOInstrumentation.cpp - MST-based PGO Instrumentation -------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements PGO instrumentation using a minimum spanning tree based
11// on the following paper:
12// [1] Donald E. Knuth, Francis R. Stevenson. Optimal measurement of points
13// for program frequency counts. BIT Numerical Mathematics 1973, Volume 13,
14// Issue 3, pp 313-322
15// The idea of the algorithm based on the fact that for each node (except for
16// the entry and exit), the sum of incoming edge counts equals the sum of
17// outgoing edge counts. The count of edge on spanning tree can be derived from
18// those edges not on the spanning tree. Knuth proves this method instruments
19// the minimum number of edges.
20//
21// The minimal spanning tree here is actually a maximum weight tree -- on-tree
22// edges have higher frequencies (more likely to execute). The idea is to
23// instrument those less frequently executed edges to reduce the runtime
24// overhead of instrumented binaries.
25//
26// This file contains two passes:
27// (1) Pass PGOInstrumentationGen which instruments the IR to generate edge
28// count profile, and generates the instrumentation for indirect call
29// profiling.
30// (2) Pass PGOInstrumentationUse which reads the edge count profile and
31// annotates the branch weights. It also reads the indirect call value
32// profiling records and annotate the indirect call instructions.
33//
34// To get the precise counter information, These two passes need to invoke at
35// the same compilation point (so they see the same IR). For pass
36// PGOInstrumentationGen, the real work is done in instrumentOneFunc(). For
37// pass PGOInstrumentationUse, the real work in done in class PGOUseFunc and
38// the profile is opened in module level and passed to each PGOUseFunc instance.
39// The shared code for PGOInstrumentationGen and PGOInstrumentationUse is put
40// in class FuncPGOInstrumentation.
41//
42// Class PGOEdge represents a CFG edge and some auxiliary information. Class
43// BBInfo contains auxiliary information for each BB. These two classes are used
44// in pass PGOInstrumentationGen. Class PGOUseEdge and UseBBInfo are the derived
45// class of PGOEdge and BBInfo, respectively. They contains extra data structure
46// used in populating profile counters.
47// The MST implementation is in Class CFGMST (CFGMST.h).
48//
49//===----------------------------------------------------------------------===//
50
51#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"
52#include "CFGMST.h"
53#include "llvm/ADT/APInt.h"
54#include "llvm/ADT/ArrayRef.h"
55#include "llvm/ADT/STLExtras.h"
56#include "llvm/ADT/SmallVector.h"
57#include "llvm/ADT/Statistic.h"
58#include "llvm/ADT/StringRef.h"
59#include "llvm/ADT/Triple.h"
60#include "llvm/ADT/Twine.h"
61#include "llvm/ADT/iterator.h"
62#include "llvm/ADT/iterator_range.h"
63#include "llvm/Analysis/BlockFrequencyInfo.h"
64#include "llvm/Analysis/BranchProbabilityInfo.h"
65#include "llvm/Analysis/CFG.h"
66#include "llvm/Analysis/IndirectCallSiteVisitor.h"
67#include "llvm/Analysis/LoopInfo.h"
68#include "llvm/Analysis/OptimizationRemarkEmitter.h"
69#include "llvm/IR/Attributes.h"
70#include "llvm/IR/BasicBlock.h"
71#include "llvm/IR/CFG.h"
72#include "llvm/IR/CallSite.h"
73#include "llvm/IR/Comdat.h"
74#include "llvm/IR/Constant.h"
75#include "llvm/IR/Constants.h"
76#include "llvm/IR/DiagnosticInfo.h"
77#include "llvm/IR/Dominators.h"
78#include "llvm/IR/Function.h"
79#include "llvm/IR/GlobalAlias.h"
80#include "llvm/IR/GlobalValue.h"
81#include "llvm/IR/GlobalVariable.h"
82#include "llvm/IR/IRBuilder.h"
83#include "llvm/IR/InstVisitor.h"
84#include "llvm/IR/InstrTypes.h"
85#include "llvm/IR/Instruction.h"
86#include "llvm/IR/Instructions.h"
87#include "llvm/IR/IntrinsicInst.h"
88#include "llvm/IR/Intrinsics.h"
89#include "llvm/IR/LLVMContext.h"
90#include "llvm/IR/MDBuilder.h"
91#include "llvm/IR/Module.h"
92#include "llvm/IR/PassManager.h"
93#include "llvm/IR/ProfileSummary.h"
94#include "llvm/IR/Type.h"
95#include "llvm/IR/Value.h"
96#include "llvm/Pass.h"
97#include "llvm/ProfileData/InstrProf.h"
98#include "llvm/ProfileData/InstrProfReader.h"
99#include "llvm/Support/BranchProbability.h"
100#include "llvm/Support/Casting.h"
101#include "llvm/Support/CommandLine.h"
102#include "llvm/Support/DOTGraphTraits.h"
103#include "llvm/Support/Debug.h"
104#include "llvm/Support/Error.h"
105#include "llvm/Support/ErrorHandling.h"
106#include "llvm/Support/GraphWriter.h"
107#include "llvm/Support/JamCRC.h"
108#include "llvm/Support/raw_ostream.h"
109#include "llvm/Transforms/Instrumentation.h"
110#include "llvm/Transforms/Utils/BasicBlockUtils.h"
111#include <algorithm>
112#include <cassert>
113#include <cstdint>
114#include <memory>
115#include <numeric>
116#include <string>
117#include <unordered_map>
118#include <utility>
119#include <vector>
120
121using namespace llvm;
122using ProfileCount = Function::ProfileCount;
123
124#define DEBUG_TYPE"pgo-instrumentation" "pgo-instrumentation"
125
126STATISTIC(NumOfPGOInstrument, "Number of edges instrumented.")static llvm::Statistic NumOfPGOInstrument = {"pgo-instrumentation"
, "NumOfPGOInstrument", "Number of edges instrumented.", {0},
{false}}
;
127STATISTIC(NumOfPGOSelectInsts, "Number of select instruction instrumented.")static llvm::Statistic NumOfPGOSelectInsts = {"pgo-instrumentation"
, "NumOfPGOSelectInsts", "Number of select instruction instrumented."
, {0}, {false}}
;
128STATISTIC(NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented.")static llvm::Statistic NumOfPGOMemIntrinsics = {"pgo-instrumentation"
, "NumOfPGOMemIntrinsics", "Number of mem intrinsics instrumented."
, {0}, {false}}
;
129STATISTIC(NumOfPGOEdge, "Number of edges.")static llvm::Statistic NumOfPGOEdge = {"pgo-instrumentation",
"NumOfPGOEdge", "Number of edges.", {0}, {false}}
;
130STATISTIC(NumOfPGOBB, "Number of basic-blocks.")static llvm::Statistic NumOfPGOBB = {"pgo-instrumentation", "NumOfPGOBB"
, "Number of basic-blocks.", {0}, {false}}
;
131STATISTIC(NumOfPGOSplit, "Number of critical edge splits.")static llvm::Statistic NumOfPGOSplit = {"pgo-instrumentation"
, "NumOfPGOSplit", "Number of critical edge splits.", {0}, {false
}}
;
132STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts.")static llvm::Statistic NumOfPGOFunc = {"pgo-instrumentation",
"NumOfPGOFunc", "Number of functions having valid profile counts."
, {0}, {false}}
;
133STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile.")static llvm::Statistic NumOfPGOMismatch = {"pgo-instrumentation"
, "NumOfPGOMismatch", "Number of functions having mismatch profile."
, {0}, {false}}
;
134STATISTIC(NumOfPGOMissing, "Number of functions without profile.")static llvm::Statistic NumOfPGOMissing = {"pgo-instrumentation"
, "NumOfPGOMissing", "Number of functions without profile.", {
0}, {false}}
;
135STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations.")static llvm::Statistic NumOfPGOICall = {"pgo-instrumentation"
, "NumOfPGOICall", "Number of indirect call value instrumentations."
, {0}, {false}}
;
136
137// Command line option to specify the file to read profile from. This is
138// mainly used for testing.
139static cl::opt<std::string>
140 PGOTestProfileFile("pgo-test-profile-file", cl::init(""), cl::Hidden,
141 cl::value_desc("filename"),
142 cl::desc("Specify the path of profile data file. This is"
143 "mainly for test purpose."));
144static cl::opt<std::string> PGOTestProfileRemappingFile(
145 "pgo-test-profile-remapping-file", cl::init(""), cl::Hidden,
146 cl::value_desc("filename"),
147 cl::desc("Specify the path of profile remapping file. This is mainly for "
148 "test purpose."));
149
150// Command line option to disable value profiling. The default is false:
151// i.e. value profiling is enabled by default. This is for debug purpose.
152static cl::opt<bool> DisableValueProfiling("disable-vp", cl::init(false),
153 cl::Hidden,
154 cl::desc("Disable Value Profiling"));
155
156// Command line option to set the maximum number of VP annotations to write to
157// the metadata for a single indirect call callsite.
158static cl::opt<unsigned> MaxNumAnnotations(
159 "icp-max-annotations", cl::init(3), cl::Hidden, cl::ZeroOrMore,
160 cl::desc("Max number of annotations for a single indirect "
161 "call callsite"));
162
163// Command line option to set the maximum number of value annotations
164// to write to the metadata for a single memop intrinsic.
165static cl::opt<unsigned> MaxNumMemOPAnnotations(
166 "memop-max-annotations", cl::init(4), cl::Hidden, cl::ZeroOrMore,
167 cl::desc("Max number of preicise value annotations for a single memop"
168 "intrinsic"));
169
170// Command line option to control appending FunctionHash to the name of a COMDAT
171// function. This is to avoid the hash mismatch caused by the preinliner.
172static cl::opt<bool> DoComdatRenaming(
173 "do-comdat-renaming", cl::init(false), cl::Hidden,
174 cl::desc("Append function hash to the name of COMDAT function to avoid "
175 "function hash mismatch due to the preinliner"));
176
177// Command line option to enable/disable the warning about missing profile
178// information.
179static cl::opt<bool>
180 PGOWarnMissing("pgo-warn-missing-function", cl::init(false), cl::Hidden,
181 cl::desc("Use this option to turn on/off "
182 "warnings about missing profile data for "
183 "functions."));
184
185// Command line option to enable/disable the warning about a hash mismatch in
186// the profile data.
187static cl::opt<bool>
188 NoPGOWarnMismatch("no-pgo-warn-mismatch", cl::init(false), cl::Hidden,
189 cl::desc("Use this option to turn off/on "
190 "warnings about profile cfg mismatch."));
191
192// Command line option to enable/disable the warning about a hash mismatch in
193// the profile data for Comdat functions, which often turns out to be false
194// positive due to the pre-instrumentation inline.
195static cl::opt<bool>
196 NoPGOWarnMismatchComdat("no-pgo-warn-mismatch-comdat", cl::init(true),
197 cl::Hidden,
198 cl::desc("The option is used to turn on/off "
199 "warnings about hash mismatch for comdat "
200 "functions."));
201
202// Command line option to enable/disable select instruction instrumentation.
203static cl::opt<bool>
204 PGOInstrSelect("pgo-instr-select", cl::init(true), cl::Hidden,
205 cl::desc("Use this option to turn on/off SELECT "
206 "instruction instrumentation. "));
207
208// Command line option to turn on CFG dot or text dump of raw profile counts
209static cl::opt<PGOViewCountsType> PGOViewRawCounts(
210 "pgo-view-raw-counts", cl::Hidden,
211 cl::desc("A boolean option to show CFG dag or text "
212 "with raw profile counts from "
213 "profile data. See also option "
214 "-pgo-view-counts. To limit graph "
215 "display to only one function, use "
216 "filtering option -view-bfi-func-name."),
217 cl::values(clEnumValN(PGOVCT_None, "none", "do not show.")llvm::cl::OptionEnumValue { "none", int(PGOVCT_None), "do not show."
}
,
218 clEnumValN(PGOVCT_Graph, "graph", "show a graph.")llvm::cl::OptionEnumValue { "graph", int(PGOVCT_Graph), "show a graph."
}
,
219 clEnumValN(PGOVCT_Text, "text", "show in text.")llvm::cl::OptionEnumValue { "text", int(PGOVCT_Text), "show in text."
}
));
220
221// Command line option to enable/disable memop intrinsic call.size profiling.
222static cl::opt<bool>
223 PGOInstrMemOP("pgo-instr-memop", cl::init(true), cl::Hidden,
224 cl::desc("Use this option to turn on/off "
225 "memory intrinsic size profiling."));
226
227// Emit branch probability as optimization remarks.
228static cl::opt<bool>
229 EmitBranchProbability("pgo-emit-branch-prob", cl::init(false), cl::Hidden,
230 cl::desc("When this option is on, the annotated "
231 "branch probability will be emitted as "
232 "optimization remarks: -{Rpass|"
233 "pass-remarks}=pgo-instrumentation"));
234
235// Command line option to turn on CFG dot dump after profile annotation.
236// Defined in Analysis/BlockFrequencyInfo.cpp: -pgo-view-counts
237extern cl::opt<PGOViewCountsType> PGOViewCounts;
238
239// Command line option to specify the name of the function for CFG dump
240// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
241extern cl::opt<std::string> ViewBlockFreqFuncName;
242
243// Return a string describing the branch condition that can be
244// used in static branch probability heuristics:
245static std::string getBranchCondString(Instruction *TI) {
246 BranchInst *BI = dyn_cast<BranchInst>(TI);
247 if (!BI || !BI->isConditional())
248 return std::string();
249
250 Value *Cond = BI->getCondition();
251 ICmpInst *CI = dyn_cast<ICmpInst>(Cond);
252 if (!CI)
253 return std::string();
254
255 std::string result;
256 raw_string_ostream OS(result);
257 OS << CmpInst::getPredicateName(CI->getPredicate()) << "_";
258 CI->getOperand(0)->getType()->print(OS, true);
259
260 Value *RHS = CI->getOperand(1);
261 ConstantInt *CV = dyn_cast<ConstantInt>(RHS);
262 if (CV) {
263 if (CV->isZero())
264 OS << "_Zero";
265 else if (CV->isOne())
266 OS << "_One";
267 else if (CV->isMinusOne())
268 OS << "_MinusOne";
269 else
270 OS << "_Const";
271 }
272 OS.flush();
273 return result;
274}
275
276namespace {
277
278/// The select instruction visitor plays three roles specified
279/// by the mode. In \c VM_counting mode, it simply counts the number of
280/// select instructions. In \c VM_instrument mode, it inserts code to count
281/// the number times TrueValue of select is taken. In \c VM_annotate mode,
282/// it reads the profile data and annotate the select instruction with metadata.
283enum VisitMode { VM_counting, VM_instrument, VM_annotate };
284class PGOUseFunc;
285
286/// Instruction Visitor class to visit select instructions.
287struct SelectInstVisitor : public InstVisitor<SelectInstVisitor> {
288 Function &F;
289 unsigned NSIs = 0; // Number of select instructions instrumented.
290 VisitMode Mode = VM_counting; // Visiting mode.
291 unsigned *CurCtrIdx = nullptr; // Pointer to current counter index.
292 unsigned TotalNumCtrs = 0; // Total number of counters
293 GlobalVariable *FuncNameVar = nullptr;
294 uint64_t FuncHash = 0;
295 PGOUseFunc *UseFunc = nullptr;
296
297 SelectInstVisitor(Function &Func) : F(Func) {}
298
299 void countSelects(Function &Func) {
300 NSIs = 0;
301 Mode = VM_counting;
302 visit(Func);
303 }
304
305 // Visit the IR stream and instrument all select instructions. \p
306 // Ind is a pointer to the counter index variable; \p TotalNC
307 // is the total number of counters; \p FNV is the pointer to the
308 // PGO function name var; \p FHash is the function hash.
309 void instrumentSelects(Function &Func, unsigned *Ind, unsigned TotalNC,
310 GlobalVariable *FNV, uint64_t FHash) {
311 Mode = VM_instrument;
312 CurCtrIdx = Ind;
313 TotalNumCtrs = TotalNC;
314 FuncHash = FHash;
315 FuncNameVar = FNV;
316 visit(Func);
317 }
318
319 // Visit the IR stream and annotate all select instructions.
320 void annotateSelects(Function &Func, PGOUseFunc *UF, unsigned *Ind) {
321 Mode = VM_annotate;
322 UseFunc = UF;
323 CurCtrIdx = Ind;
324 visit(Func);
325 }
326
327 void instrumentOneSelectInst(SelectInst &SI);
328 void annotateOneSelectInst(SelectInst &SI);
329
330 // Visit \p SI instruction and perform tasks according to visit mode.
331 void visitSelectInst(SelectInst &SI);
332
333 // Return the number of select instructions. This needs be called after
334 // countSelects().
335 unsigned getNumOfSelectInsts() const { return NSIs; }
336};
337
338/// Instruction Visitor class to visit memory intrinsic calls.
339struct MemIntrinsicVisitor : public InstVisitor<MemIntrinsicVisitor> {
340 Function &F;
341 unsigned NMemIs = 0; // Number of memIntrinsics instrumented.
342 VisitMode Mode = VM_counting; // Visiting mode.
343 unsigned CurCtrId = 0; // Current counter index.
344 unsigned TotalNumCtrs = 0; // Total number of counters
345 GlobalVariable *FuncNameVar = nullptr;
346 uint64_t FuncHash = 0;
347 PGOUseFunc *UseFunc = nullptr;
348 std::vector<Instruction *> Candidates;
349
350 MemIntrinsicVisitor(Function &Func) : F(Func) {}
351
352 void countMemIntrinsics(Function &Func) {
353 NMemIs = 0;
354 Mode = VM_counting;
355 visit(Func);
356 }
357
358 void instrumentMemIntrinsics(Function &Func, unsigned TotalNC,
359 GlobalVariable *FNV, uint64_t FHash) {
360 Mode = VM_instrument;
361 TotalNumCtrs = TotalNC;
362 FuncHash = FHash;
363 FuncNameVar = FNV;
364 visit(Func);
365 }
366
367 std::vector<Instruction *> findMemIntrinsics(Function &Func) {
368 Candidates.clear();
369 Mode = VM_annotate;
370 visit(Func);
371 return Candidates;
372 }
373
374 // Visit the IR stream and annotate all mem intrinsic call instructions.
375 void instrumentOneMemIntrinsic(MemIntrinsic &MI);
376
377 // Visit \p MI instruction and perform tasks according to visit mode.
378 void visitMemIntrinsic(MemIntrinsic &SI);
379
380 unsigned getNumOfMemIntrinsics() const { return NMemIs; }
381};
382
383class PGOInstrumentationGenLegacyPass : public ModulePass {
384public:
385 static char ID;
386
387 PGOInstrumentationGenLegacyPass() : ModulePass(ID) {
388 initializePGOInstrumentationGenLegacyPassPass(
389 *PassRegistry::getPassRegistry());
390 }
391
392 StringRef getPassName() const override { return "PGOInstrumentationGenPass"; }
393
394private:
395 bool runOnModule(Module &M) override;
396
397 void getAnalysisUsage(AnalysisUsage &AU) const override {
398 AU.addRequired<BlockFrequencyInfoWrapperPass>();
399 }
400};
401
402class PGOInstrumentationUseLegacyPass : public ModulePass {
403public:
404 static char ID;
405
406 // Provide the profile filename as the parameter.
407 PGOInstrumentationUseLegacyPass(std::string Filename = "")
408 : ModulePass(ID), ProfileFileName(std::move(Filename)) {
409 if (!PGOTestProfileFile.empty())
410 ProfileFileName = PGOTestProfileFile;
411 initializePGOInstrumentationUseLegacyPassPass(
412 *PassRegistry::getPassRegistry());
413 }
414
415 StringRef getPassName() const override { return "PGOInstrumentationUsePass"; }
416
417private:
418 std::string ProfileFileName;
419
420 bool runOnModule(Module &M) override;
421
422 void getAnalysisUsage(AnalysisUsage &AU) const override {
423 AU.addRequired<BlockFrequencyInfoWrapperPass>();
424 }
425};
426
427} // end anonymous namespace
428
429char PGOInstrumentationGenLegacyPass::ID = 0;
430
431INITIALIZE_PASS_BEGIN(PGOInstrumentationGenLegacyPass, "pgo-instr-gen",static void *initializePGOInstrumentationGenLegacyPassPassOnce
(PassRegistry &Registry) {
432 "PGO instrumentation.", false, false)static void *initializePGOInstrumentationGenLegacyPassPassOnce
(PassRegistry &Registry) {
433INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)initializeBlockFrequencyInfoWrapperPassPass(Registry);
434INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)initializeBranchProbabilityInfoWrapperPassPass(Registry);
435INITIALIZE_PASS_END(PGOInstrumentationGenLegacyPass, "pgo-instr-gen",PassInfo *PI = new PassInfo( "PGO instrumentation.", "pgo-instr-gen"
, &PGOInstrumentationGenLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<PGOInstrumentationGenLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializePGOInstrumentationGenLegacyPassPassFlag
; void llvm::initializePGOInstrumentationGenLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializePGOInstrumentationGenLegacyPassPassFlag
, initializePGOInstrumentationGenLegacyPassPassOnce, std::ref
(Registry)); }
436 "PGO instrumentation.", false, false)PassInfo *PI = new PassInfo( "PGO instrumentation.", "pgo-instr-gen"
, &PGOInstrumentationGenLegacyPass::ID, PassInfo::NormalCtor_t
(callDefaultCtor<PGOInstrumentationGenLegacyPass>), false
, false); Registry.registerPass(*PI, true); return PI; } static
llvm::once_flag InitializePGOInstrumentationGenLegacyPassPassFlag
; void llvm::initializePGOInstrumentationGenLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializePGOInstrumentationGenLegacyPassPassFlag
, initializePGOInstrumentationGenLegacyPassPassOnce, std::ref
(Registry)); }
437
438ModulePass *llvm::createPGOInstrumentationGenLegacyPass() {
439 return new PGOInstrumentationGenLegacyPass();
440}
441
442char PGOInstrumentationUseLegacyPass::ID = 0;
443
444INITIALIZE_PASS_BEGIN(PGOInstrumentationUseLegacyPass, "pgo-instr-use",static void *initializePGOInstrumentationUseLegacyPassPassOnce
(PassRegistry &Registry) {
445 "Read PGO instrumentation profile.", false, false)static void *initializePGOInstrumentationUseLegacyPassPassOnce
(PassRegistry &Registry) {
446INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)initializeBlockFrequencyInfoWrapperPassPass(Registry);
447INITIALIZE_PASS_DEPENDENCY(BranchProbabilityInfoWrapperPass)initializeBranchProbabilityInfoWrapperPassPass(Registry);
448INITIALIZE_PASS_END(PGOInstrumentationUseLegacyPass, "pgo-instr-use",PassInfo *PI = new PassInfo( "Read PGO instrumentation profile."
, "pgo-instr-use", &PGOInstrumentationUseLegacyPass::ID, PassInfo
::NormalCtor_t(callDefaultCtor<PGOInstrumentationUseLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializePGOInstrumentationUseLegacyPassPassFlag
; void llvm::initializePGOInstrumentationUseLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializePGOInstrumentationUseLegacyPassPassFlag
, initializePGOInstrumentationUseLegacyPassPassOnce, std::ref
(Registry)); }
449 "Read PGO instrumentation profile.", false, false)PassInfo *PI = new PassInfo( "Read PGO instrumentation profile."
, "pgo-instr-use", &PGOInstrumentationUseLegacyPass::ID, PassInfo
::NormalCtor_t(callDefaultCtor<PGOInstrumentationUseLegacyPass
>), false, false); Registry.registerPass(*PI, true); return
PI; } static llvm::once_flag InitializePGOInstrumentationUseLegacyPassPassFlag
; void llvm::initializePGOInstrumentationUseLegacyPassPass(PassRegistry
&Registry) { llvm::call_once(InitializePGOInstrumentationUseLegacyPassPassFlag
, initializePGOInstrumentationUseLegacyPassPassOnce, std::ref
(Registry)); }
450
451ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename) {
452 return new PGOInstrumentationUseLegacyPass(Filename.str());
453}
454
455namespace {
456
457/// An MST based instrumentation for PGO
458///
459/// Implements a Minimum Spanning Tree (MST) based instrumentation for PGO
460/// in the function level.
461struct PGOEdge {
462 // This class implements the CFG edges. Note the CFG can be a multi-graph.
463 // So there might be multiple edges with same SrcBB and DestBB.
464 const BasicBlock *SrcBB;
465 const BasicBlock *DestBB;
466 uint64_t Weight;
467 bool InMST = false;
468 bool Removed = false;
469 bool IsCritical = false;
470
471 PGOEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1)
472 : SrcBB(Src), DestBB(Dest), Weight(W) {}
473
474 // Return the information string of an edge.
475 const std::string infoString() const {
476 return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") +
477 (IsCritical ? "c" : " ") + " W=" + Twine(Weight)).str();
478 }
479};
480
481// This class stores the auxiliary information for each BB.
482struct BBInfo {
483 BBInfo *Group;
484 uint32_t Index;
485 uint32_t Rank = 0;
486
487 BBInfo(unsigned IX) : Group(this), Index(IX) {}
488
489 // Return the information string of this object.
490 const std::string infoString() const {
491 return (Twine("Index=") + Twine(Index)).str();
492 }
493};
494
495// This class implements the CFG edges. Note the CFG can be a multi-graph.
496template <class Edge, class BBInfo> class FuncPGOInstrumentation {
497private:
498 Function &F;
499
500 // A map that stores the Comdat group in function F.
501 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
502
503 void computeCFGHash();
504 void renameComdatFunction();
505
506public:
507 std::vector<std::vector<Instruction *>> ValueSites;
508 SelectInstVisitor SIVisitor;
509 MemIntrinsicVisitor MIVisitor;
510 std::string FuncName;
511 GlobalVariable *FuncNameVar;
512
513 // CFG hash value for this function.
514 uint64_t FunctionHash = 0;
515
516 // The Minimum Spanning Tree of function CFG.
517 CFGMST<Edge, BBInfo> MST;
518
519 // Give an edge, find the BB that will be instrumented.
520 // Return nullptr if there is no BB to be instrumented.
521 BasicBlock *getInstrBB(Edge *E);
522
523 // Return the auxiliary BB information.
524 BBInfo &getBBInfo(const BasicBlock *BB) const { return MST.getBBInfo(BB); }
525
526 // Return the auxiliary BB information if available.
527 BBInfo *findBBInfo(const BasicBlock *BB) const { return MST.findBBInfo(BB); }
528
529 // Dump edges and BB information.
530 void dumpInfo(std::string Str = "") const {
531 MST.dumpEdges(dbgs(), Twine("Dump Function ") + FuncName + " Hash: " +
532 Twine(FunctionHash) + "\t" + Str);
533 }
534
535 FuncPGOInstrumentation(
536 Function &Func,
537 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
538 bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr,
539 BlockFrequencyInfo *BFI = nullptr)
540 : F(Func), ComdatMembers(ComdatMembers), ValueSites(IPVK_Last + 1),
541 SIVisitor(Func), MIVisitor(Func), MST(F, BPI, BFI) {
542 // This should be done before CFG hash computation.
543 SIVisitor.countSelects(Func);
544 MIVisitor.countMemIntrinsics(Func);
545 NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
546 NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics();
547 ValueSites[IPVK_IndirectCallTarget] = findIndirectCallSites(Func);
548 ValueSites[IPVK_MemOPSize] = MIVisitor.findMemIntrinsics(Func);
549
550 FuncName = getPGOFuncName(F);
551 computeCFGHash();
552 if (!ComdatMembers.empty())
553 renameComdatFunction();
554 LLVM_DEBUG(dumpInfo("after CFGMST"))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dumpInfo("after CFGMST"); } } while
(false)
;
555
556 NumOfPGOBB += MST.BBInfos.size();
557 for (auto &E : MST.AllEdges) {
558 if (E->Removed)
559 continue;
560 NumOfPGOEdge++;
561 if (!E->InMST)
562 NumOfPGOInstrument++;
563 }
564
565 if (CreateGlobalVar)
566 FuncNameVar = createPGOFuncNameVar(F, FuncName);
567 }
568
569 // Return the number of profile counters needed for the function.
570 unsigned getNumCounters() {
571 unsigned NumCounters = 0;
572 for (auto &E : this->MST.AllEdges) {
573 if (!E->InMST && !E->Removed)
574 NumCounters++;
575 }
576 return NumCounters + SIVisitor.getNumOfSelectInsts();
577 }
578};
579
580} // end anonymous namespace
581
582// Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
583// value of each BB in the CFG. The higher 32 bits record the number of edges.
584template <class Edge, class BBInfo>
585void FuncPGOInstrumentation<Edge, BBInfo>::computeCFGHash() {
586 std::vector<char> Indexes;
587 JamCRC JC;
588 for (auto &BB : F) {
589 const Instruction *TI = BB.getTerminator();
590 for (unsigned I = 0, E = TI->getNumSuccessors(); I != E; ++I) {
591 BasicBlock *Succ = TI->getSuccessor(I);
592 auto BI = findBBInfo(Succ);
593 if (BI == nullptr)
594 continue;
595 uint32_t Index = BI->Index;
596 for (int J = 0; J < 4; J++)
597 Indexes.push_back((char)(Index >> (J * 8)));
598 }
599 }
600 JC.update(Indexes);
601 FunctionHash = (uint64_t)SIVisitor.getNumOfSelectInsts() << 56 |
602 (uint64_t)ValueSites[IPVK_IndirectCallTarget].size() << 48 |
603 (uint64_t)MST.AllEdges.size() << 32 | JC.getCRC();
604 LLVM_DEBUG(dbgs() << "Function Hash Computation for " << F.getName() << ":\n"do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false)
605 << " CRC = " << JC.getCRC()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false)
606 << ", Selects = " << SIVisitor.getNumOfSelectInsts()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false)
607 << ", Edges = " << MST.AllEdges.size() << ", ICSites = "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false)
608 << ValueSites[IPVK_IndirectCallTarget].size()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false)
609 << ", Hash = " << FunctionHash << "\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Function Hash Computation for "
<< F.getName() << ":\n" << " CRC = " <<
JC.getCRC() << ", Selects = " << SIVisitor.getNumOfSelectInsts
() << ", Edges = " << MST.AllEdges.size() <<
", ICSites = " << ValueSites[IPVK_IndirectCallTarget].
size() << ", Hash = " << FunctionHash << "\n"
;; } } while (false)
;
610}
611
612// Check if we can safely rename this Comdat function.
613static bool canRenameComdat(
614 Function &F,
615 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
616 if (!DoComdatRenaming || !canRenameComdatFunc(F, true))
617 return false;
618
619 // FIXME: Current only handle those Comdat groups that only containing one
620 // function and function aliases.
621 // (1) For a Comdat group containing multiple functions, we need to have a
622 // unique postfix based on the hashes for each function. There is a
623 // non-trivial code refactoring to do this efficiently.
624 // (2) Variables can not be renamed, so we can not rename Comdat function in a
625 // group including global vars.
626 Comdat *C = F.getComdat();
627 for (auto &&CM : make_range(ComdatMembers.equal_range(C))) {
628 if (dyn_cast<GlobalAlias>(CM.second))
629 continue;
630 Function *FM = dyn_cast<Function>(CM.second);
631 if (FM != &F)
632 return false;
633 }
634 return true;
635}
636
637// Append the CFGHash to the Comdat function name.
638template <class Edge, class BBInfo>
639void FuncPGOInstrumentation<Edge, BBInfo>::renameComdatFunction() {
640 if (!canRenameComdat(F, ComdatMembers))
641 return;
642 std::string OrigName = F.getName().str();
643 std::string NewFuncName =
644 Twine(F.getName() + "." + Twine(FunctionHash)).str();
645 F.setName(Twine(NewFuncName));
646 GlobalAlias::create(GlobalValue::WeakAnyLinkage, OrigName, &F);
647 FuncName = Twine(FuncName + "." + Twine(FunctionHash)).str();
648 Comdat *NewComdat;
649 Module *M = F.getParent();
650 // For AvailableExternallyLinkage functions, change the linkage to
651 // LinkOnceODR and put them into comdat. This is because after renaming, there
652 // is no backup external copy available for the function.
653 if (!F.hasComdat()) {
654 assert(F.getLinkage() == GlobalValue::AvailableExternallyLinkage)((F.getLinkage() == GlobalValue::AvailableExternallyLinkage) ?
static_cast<void> (0) : __assert_fail ("F.getLinkage() == GlobalValue::AvailableExternallyLinkage"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 654, __PRETTY_FUNCTION__))
;
655 NewComdat = M->getOrInsertComdat(StringRef(NewFuncName));
656 F.setLinkage(GlobalValue::LinkOnceODRLinkage);
657 F.setComdat(NewComdat);
658 return;
659 }
660
661 // This function belongs to a single function Comdat group.
662 Comdat *OrigComdat = F.getComdat();
663 std::string NewComdatName =
664 Twine(OrigComdat->getName() + "." + Twine(FunctionHash)).str();
665 NewComdat = M->getOrInsertComdat(StringRef(NewComdatName));
666 NewComdat->setSelectionKind(OrigComdat->getSelectionKind());
667
668 for (auto &&CM : make_range(ComdatMembers.equal_range(OrigComdat))) {
669 if (GlobalAlias *GA = dyn_cast<GlobalAlias>(CM.second)) {
670 // For aliases, change the name directly.
671 assert(dyn_cast<Function>(GA->getAliasee()->stripPointerCasts()) == &F)((dyn_cast<Function>(GA->getAliasee()->stripPointerCasts
()) == &F) ? static_cast<void> (0) : __assert_fail (
"dyn_cast<Function>(GA->getAliasee()->stripPointerCasts()) == &F"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 671, __PRETTY_FUNCTION__))
;
672 std::string OrigGAName = GA->getName().str();
673 GA->setName(Twine(GA->getName() + "." + Twine(FunctionHash)));
674 GlobalAlias::create(GlobalValue::WeakAnyLinkage, OrigGAName, GA);
675 continue;
676 }
677 // Must be a function.
678 Function *CF = dyn_cast<Function>(CM.second);
679 assert(CF)((CF) ? static_cast<void> (0) : __assert_fail ("CF", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 679, __PRETTY_FUNCTION__))
;
680 CF->setComdat(NewComdat);
681 }
682}
683
684// Given a CFG E to be instrumented, find which BB to place the instrumented
685// code. The function will split the critical edge if necessary.
686template <class Edge, class BBInfo>
687BasicBlock *FuncPGOInstrumentation<Edge, BBInfo>::getInstrBB(Edge *E) {
688 if (E->InMST || E->Removed)
689 return nullptr;
690
691 BasicBlock *SrcBB = const_cast<BasicBlock *>(E->SrcBB);
692 BasicBlock *DestBB = const_cast<BasicBlock *>(E->DestBB);
693 // For a fake edge, instrument the real BB.
694 if (SrcBB == nullptr)
695 return DestBB;
696 if (DestBB == nullptr)
697 return SrcBB;
698
699 // Instrument the SrcBB if it has a single successor,
700 // otherwise, the DestBB if this is not a critical edge.
701 Instruction *TI = SrcBB->getTerminator();
702 if (TI->getNumSuccessors() <= 1)
703 return SrcBB;
704 if (!E->IsCritical)
705 return DestBB;
706
707 // For a critical edge, we have to split. Instrument the newly
708 // created BB.
709 NumOfPGOSplit++;
710 LLVM_DEBUG(dbgs() << "Split critical edge: " << getBBInfo(SrcBB).Indexdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Split critical edge: "
<< getBBInfo(SrcBB).Index << " --> " <<
getBBInfo(DestBB).Index << "\n"; } } while (false)
711 << " --> " << getBBInfo(DestBB).Index << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Split critical edge: "
<< getBBInfo(SrcBB).Index << " --> " <<
getBBInfo(DestBB).Index << "\n"; } } while (false)
;
712 unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
713 BasicBlock *InstrBB = SplitCriticalEdge(TI, SuccNum);
714 assert(InstrBB && "Critical edge is not split")((InstrBB && "Critical edge is not split") ? static_cast
<void> (0) : __assert_fail ("InstrBB && \"Critical edge is not split\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 714, __PRETTY_FUNCTION__))
;
715
716 E->Removed = true;
717 return InstrBB;
718}
719
720// Visit all edge and instrument the edges not in MST, and do value profiling.
721// Critical edges will be split.
722static void instrumentOneFunc(
723 Function &F, Module *M, BranchProbabilityInfo *BPI, BlockFrequencyInfo *BFI,
724 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
725 // Split indirectbr critical edges here before computing the MST rather than
726 // later in getInstrBB() to avoid invalidating it.
727 SplitIndirectBrCriticalEdges(F, BPI, BFI);
728 FuncPGOInstrumentation<PGOEdge, BBInfo> FuncInfo(F, ComdatMembers, true, BPI,
729 BFI);
730 unsigned NumCounters = FuncInfo.getNumCounters();
731
732 uint32_t I = 0;
733 Type *I8PtrTy = Type::getInt8PtrTy(M->getContext());
734 for (auto &E : FuncInfo.MST.AllEdges) {
735 BasicBlock *InstrBB = FuncInfo.getInstrBB(E.get());
736 if (!InstrBB)
737 continue;
738
739 IRBuilder<> Builder(InstrBB, InstrBB->getFirstInsertionPt());
740 assert(Builder.GetInsertPoint() != InstrBB->end() &&((Builder.GetInsertPoint() != InstrBB->end() && "Cannot get the Instrumentation point"
) ? static_cast<void> (0) : __assert_fail ("Builder.GetInsertPoint() != InstrBB->end() && \"Cannot get the Instrumentation point\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 741, __PRETTY_FUNCTION__))
741 "Cannot get the Instrumentation point")((Builder.GetInsertPoint() != InstrBB->end() && "Cannot get the Instrumentation point"
) ? static_cast<void> (0) : __assert_fail ("Builder.GetInsertPoint() != InstrBB->end() && \"Cannot get the Instrumentation point\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 741, __PRETTY_FUNCTION__))
;
742 Builder.CreateCall(
743 Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment),
744 {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
745 Builder.getInt64(FuncInfo.FunctionHash), Builder.getInt32(NumCounters),
746 Builder.getInt32(I++)});
747 }
748
749 // Now instrument select instructions:
750 FuncInfo.SIVisitor.instrumentSelects(F, &I, NumCounters, FuncInfo.FuncNameVar,
751 FuncInfo.FunctionHash);
752 assert(I == NumCounters)((I == NumCounters) ? static_cast<void> (0) : __assert_fail
("I == NumCounters", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 752, __PRETTY_FUNCTION__))
;
753
754 if (DisableValueProfiling)
755 return;
756
757 unsigned NumIndirectCallSites = 0;
758 for (auto &I : FuncInfo.ValueSites[IPVK_IndirectCallTarget]) {
759 CallSite CS(I);
760 Value *Callee = CS.getCalledValue();
761 LLVM_DEBUG(dbgs() << "Instrument one indirect call: CallSite Index = "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Instrument one indirect call: CallSite Index = "
<< NumIndirectCallSites << "\n"; } } while (false
)
762 << NumIndirectCallSites << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Instrument one indirect call: CallSite Index = "
<< NumIndirectCallSites << "\n"; } } while (false
)
;
763 IRBuilder<> Builder(I);
764 assert(Builder.GetInsertPoint() != I->getParent()->end() &&((Builder.GetInsertPoint() != I->getParent()->end() &&
"Cannot get the Instrumentation point") ? static_cast<void
> (0) : __assert_fail ("Builder.GetInsertPoint() != I->getParent()->end() && \"Cannot get the Instrumentation point\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 765, __PRETTY_FUNCTION__))
765 "Cannot get the Instrumentation point")((Builder.GetInsertPoint() != I->getParent()->end() &&
"Cannot get the Instrumentation point") ? static_cast<void
> (0) : __assert_fail ("Builder.GetInsertPoint() != I->getParent()->end() && \"Cannot get the Instrumentation point\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 765, __PRETTY_FUNCTION__))
;
766 Builder.CreateCall(
767 Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
768 {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
769 Builder.getInt64(FuncInfo.FunctionHash),
770 Builder.CreatePtrToInt(Callee, Builder.getInt64Ty()),
771 Builder.getInt32(IPVK_IndirectCallTarget),
772 Builder.getInt32(NumIndirectCallSites++)});
773 }
774 NumOfPGOICall += NumIndirectCallSites;
775
776 // Now instrument memop intrinsic calls.
777 FuncInfo.MIVisitor.instrumentMemIntrinsics(
778 F, NumCounters, FuncInfo.FuncNameVar, FuncInfo.FunctionHash);
779}
780
781namespace {
782
783// This class represents a CFG edge in profile use compilation.
784struct PGOUseEdge : public PGOEdge {
785 bool CountValid = false;
786 uint64_t CountValue = 0;
787
788 PGOUseEdge(const BasicBlock *Src, const BasicBlock *Dest, uint64_t W = 1)
789 : PGOEdge(Src, Dest, W) {}
790
791 // Set edge count value
792 void setEdgeCount(uint64_t Value) {
793 CountValue = Value;
794 CountValid = true;
795 }
796
797 // Return the information string for this object.
798 const std::string infoString() const {
799 if (!CountValid)
800 return PGOEdge::infoString();
801 return (Twine(PGOEdge::infoString()) + " Count=" + Twine(CountValue))
802 .str();
803 }
804};
805
806using DirectEdges = SmallVector<PGOUseEdge *, 2>;
807
808// This class stores the auxiliary information for each BB.
809struct UseBBInfo : public BBInfo {
810 uint64_t CountValue = 0;
811 bool CountValid;
812 int32_t UnknownCountInEdge = 0;
813 int32_t UnknownCountOutEdge = 0;
814 DirectEdges InEdges;
815 DirectEdges OutEdges;
816
817 UseBBInfo(unsigned IX) : BBInfo(IX), CountValid(false) {}
818
819 UseBBInfo(unsigned IX, uint64_t C)
820 : BBInfo(IX), CountValue(C), CountValid(true) {}
821
822 // Set the profile count value for this BB.
823 void setBBInfoCount(uint64_t Value) {
824 CountValue = Value;
825 CountValid = true;
826 }
827
828 // Return the information string of this object.
829 const std::string infoString() const {
830 if (!CountValid)
831 return BBInfo::infoString();
832 return (Twine(BBInfo::infoString()) + " Count=" + Twine(CountValue)).str();
833 }
834};
835
836} // end anonymous namespace
837
838// Sum up the count values for all the edges.
839static uint64_t sumEdgeCount(const ArrayRef<PGOUseEdge *> Edges) {
840 uint64_t Total = 0;
841 for (auto &E : Edges) {
842 if (E->Removed)
843 continue;
844 Total += E->CountValue;
845 }
846 return Total;
847}
848
849namespace {
850
851class PGOUseFunc {
852public:
853 PGOUseFunc(Function &Func, Module *Modu,
854 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers,
855 BranchProbabilityInfo *BPI = nullptr,
856 BlockFrequencyInfo *BFIin = nullptr)
857 : F(Func), M(Modu), BFI(BFIin),
858 FuncInfo(Func, ComdatMembers, false, BPI, BFIin),
859 FreqAttr(FFA_Normal) {}
860
861 // Read counts for the instrumented BB from profile.
862 bool readCounters(IndexedInstrProfReader *PGOReader);
863
864 // Populate the counts for all BBs.
865 void populateCounters();
866
867 // Set the branch weights based on the count values.
868 void setBranchWeights();
869
870 // Annotate the value profile call sites for all value kind.
871 void annotateValueSites();
872
873 // Annotate the value profile call sites for one value kind.
874 void annotateValueSites(uint32_t Kind);
875
876 // Annotate the irreducible loop header weights.
877 void annotateIrrLoopHeaderWeights();
878
879 // The hotness of the function from the profile count.
880 enum FuncFreqAttr { FFA_Normal, FFA_Cold, FFA_Hot };
881
882 // Return the function hotness from the profile.
883 FuncFreqAttr getFuncFreqAttr() const { return FreqAttr; }
884
885 // Return the function hash.
886 uint64_t getFuncHash() const { return FuncInfo.FunctionHash; }
887
888 // Return the profile record for this function;
889 InstrProfRecord &getProfileRecord() { return ProfileRecord; }
890
891 // Return the auxiliary BB information.
892 UseBBInfo &getBBInfo(const BasicBlock *BB) const {
893 return FuncInfo.getBBInfo(BB);
894 }
895
896 // Return the auxiliary BB information if available.
897 UseBBInfo *findBBInfo(const BasicBlock *BB) const {
898 return FuncInfo.findBBInfo(BB);
899 }
900
901 Function &getFunc() const { return F; }
902
903 void dumpInfo(std::string Str = "") const {
904 FuncInfo.dumpInfo(Str);
905 }
906
907private:
908 Function &F;
909 Module *M;
910 BlockFrequencyInfo *BFI;
911
912 // This member stores the shared information with class PGOGenFunc.
913 FuncPGOInstrumentation<PGOUseEdge, UseBBInfo> FuncInfo;
914
915 // The maximum count value in the profile. This is only used in PGO use
916 // compilation.
917 uint64_t ProgramMaxCount;
918
919 // Position of counter that remains to be read.
920 uint32_t CountPosition = 0;
921
922 // Total size of the profile count for this function.
923 uint32_t ProfileCountSize = 0;
924
925 // ProfileRecord for this function.
926 InstrProfRecord ProfileRecord;
927
928 // Function hotness info derived from profile.
929 FuncFreqAttr FreqAttr;
930
931 // Find the Instrumented BB and set the value.
932 void setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);
933
934 // Set the edge counter value for the unknown edge -- there should be only
935 // one unknown edge.
936 void setEdgeCount(DirectEdges &Edges, uint64_t Value);
937
938 // Return FuncName string;
939 const std::string getFuncName() const { return FuncInfo.FuncName; }
940
941 // Set the hot/cold inline hints based on the count values.
942 // FIXME: This function should be removed once the functionality in
943 // the inliner is implemented.
944 void markFunctionAttributes(uint64_t EntryCount, uint64_t MaxCount) {
945 if (ProgramMaxCount == 0)
946 return;
947 // Threshold of the hot functions.
948 const BranchProbability HotFunctionThreshold(1, 100);
949 // Threshold of the cold functions.
950 const BranchProbability ColdFunctionThreshold(2, 10000);
951 if (EntryCount >= HotFunctionThreshold.scale(ProgramMaxCount))
952 FreqAttr = FFA_Hot;
953 else if (MaxCount <= ColdFunctionThreshold.scale(ProgramMaxCount))
954 FreqAttr = FFA_Cold;
955 }
956};
957
958} // end anonymous namespace
959
960// Visit all the edges and assign the count value for the instrumented
961// edges and the BB.
962void PGOUseFunc::setInstrumentedCounts(
963 const std::vector<uint64_t> &CountFromProfile) {
964 assert(FuncInfo.getNumCounters() == CountFromProfile.size())((FuncInfo.getNumCounters() == CountFromProfile.size()) ? static_cast
<void> (0) : __assert_fail ("FuncInfo.getNumCounters() == CountFromProfile.size()"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 964, __PRETTY_FUNCTION__))
;
965 // Use a worklist as we will update the vector during the iteration.
966 std::vector<PGOUseEdge *> WorkList;
967 for (auto &E : FuncInfo.MST.AllEdges)
968 WorkList.push_back(E.get());
969
970 uint32_t I = 0;
971 for (auto &E : WorkList) {
972 BasicBlock *InstrBB = FuncInfo.getInstrBB(E);
973 if (!InstrBB)
974 continue;
975 uint64_t CountValue = CountFromProfile[I++];
976 if (!E->Removed) {
977 getBBInfo(InstrBB).setBBInfoCount(CountValue);
978 E->setEdgeCount(CountValue);
979 continue;
980 }
981
982 // Need to add two new edges.
983 BasicBlock *SrcBB = const_cast<BasicBlock *>(E->SrcBB);
984 BasicBlock *DestBB = const_cast<BasicBlock *>(E->DestBB);
985 // Add new edge of SrcBB->InstrBB.
986 PGOUseEdge &NewEdge = FuncInfo.MST.addEdge(SrcBB, InstrBB, 0);
987 NewEdge.setEdgeCount(CountValue);
988 // Add new edge of InstrBB->DestBB.
989 PGOUseEdge &NewEdge1 = FuncInfo.MST.addEdge(InstrBB, DestBB, 0);
990 NewEdge1.setEdgeCount(CountValue);
991 NewEdge1.InMST = true;
992 getBBInfo(InstrBB).setBBInfoCount(CountValue);
993 }
994 ProfileCountSize = CountFromProfile.size();
995 CountPosition = I;
996}
997
998// Set the count value for the unknown edge. There should be one and only one
999// unknown edge in Edges vector.
1000void PGOUseFunc::setEdgeCount(DirectEdges &Edges, uint64_t Value) {
1001 for (auto &E : Edges) {
1002 if (E->CountValid)
1003 continue;
1004 E->setEdgeCount(Value);
1005
1006 getBBInfo(E->SrcBB).UnknownCountOutEdge--;
1007 getBBInfo(E->DestBB).UnknownCountInEdge--;
1008 return;
1009 }
1010 llvm_unreachable("Cannot find the unknown count edge")::llvm::llvm_unreachable_internal("Cannot find the unknown count edge"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1010)
;
1011}
1012
1013// Read the profile from ProfileFileName and assign the value to the
1014// instrumented BB and the edges. This function also updates ProgramMaxCount.
1015// Return true if the profile are successfully read, and false on errors.
1016bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader) {
1017 auto &Ctx = M->getContext();
1018 Expected<InstrProfRecord> Result =
1019 PGOReader->getInstrProfRecord(FuncInfo.FuncName, FuncInfo.FunctionHash);
1020 if (Error E = Result.takeError()) {
11
Taking true branch
1021 handleAllErrors(std::move(E), [&](const InstrProfError &IPE) {
12
Calling 'handleAllErrors<(lambda at /build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp:1021:35)>'
1022 auto Err = IPE.get();
1023 bool SkipWarning = false;
1024 if (Err == instrprof_error::unknown_function) {
1025 NumOfPGOMissing++;
1026 SkipWarning = !PGOWarnMissing;
1027 } else if (Err == instrprof_error::hash_mismatch ||
1028 Err == instrprof_error::malformed) {
1029 NumOfPGOMismatch++;
1030 SkipWarning =
1031 NoPGOWarnMismatch ||
1032 (NoPGOWarnMismatchComdat &&
1033 (F.hasComdat() ||
1034 F.getLinkage() == GlobalValue::AvailableExternallyLinkage));
1035 }
1036
1037 if (SkipWarning)
1038 return;
1039
1040 std::string Msg = IPE.message() + std::string(" ") + F.getName().str();
1041 Ctx.diagnose(
1042 DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
1043 });
1044 return false;
1045 }
1046 ProfileRecord = std::move(Result.get());
1047 std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts;
1048
1049 NumOfPGOFunc++;
1050 LLVM_DEBUG(dbgs() << CountFromProfile.size() << " counts\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << CountFromProfile.size
() << " counts\n"; } } while (false)
;
1051 uint64_t ValueSum = 0;
1052 for (unsigned I = 0, S = CountFromProfile.size(); I < S; I++) {
1053 LLVM_DEBUG(dbgs() << " " << I << ": " << CountFromProfile[I] << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << " " << I <<
": " << CountFromProfile[I] << "\n"; } } while (
false)
;
1054 ValueSum += CountFromProfile[I];
1055 }
1056
1057 LLVM_DEBUG(dbgs() << "SUM = " << ValueSum << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "SUM = " <<
ValueSum << "\n"; } } while (false)
;
1058
1059 getBBInfo(nullptr).UnknownCountOutEdge = 2;
1060 getBBInfo(nullptr).UnknownCountInEdge = 2;
1061
1062 setInstrumentedCounts(CountFromProfile);
1063 ProgramMaxCount = PGOReader->getMaximumFunctionCount();
1064 return true;
1065}
1066
1067// Populate the counters from instrumented BBs to all BBs.
1068// In the end of this operation, all BBs should have a valid count value.
1069void PGOUseFunc::populateCounters() {
1070 // First set up Count variable for all BBs.
1071 for (auto &E : FuncInfo.MST.AllEdges) {
1072 if (E->Removed)
1073 continue;
1074
1075 const BasicBlock *SrcBB = E->SrcBB;
1076 const BasicBlock *DestBB = E->DestBB;
1077 UseBBInfo &SrcInfo = getBBInfo(SrcBB);
1078 UseBBInfo &DestInfo = getBBInfo(DestBB);
1079 SrcInfo.OutEdges.push_back(E.get());
1080 DestInfo.InEdges.push_back(E.get());
1081 SrcInfo.UnknownCountOutEdge++;
1082 DestInfo.UnknownCountInEdge++;
1083
1084 if (!E->CountValid)
1085 continue;
1086 DestInfo.UnknownCountInEdge--;
1087 SrcInfo.UnknownCountOutEdge--;
1088 }
1089
1090 bool Changes = true;
1091 unsigned NumPasses = 0;
1092 while (Changes) {
1093 NumPasses++;
1094 Changes = false;
1095
1096 // For efficient traversal, it's better to start from the end as most
1097 // of the instrumented edges are at the end.
1098 for (auto &BB : reverse(F)) {
1099 UseBBInfo *Count = findBBInfo(&BB);
1100 if (Count == nullptr)
1101 continue;
1102 if (!Count->CountValid) {
1103 if (Count->UnknownCountOutEdge == 0) {
1104 Count->CountValue = sumEdgeCount(Count->OutEdges);
1105 Count->CountValid = true;
1106 Changes = true;
1107 } else if (Count->UnknownCountInEdge == 0) {
1108 Count->CountValue = sumEdgeCount(Count->InEdges);
1109 Count->CountValid = true;
1110 Changes = true;
1111 }
1112 }
1113 if (Count->CountValid) {
1114 if (Count->UnknownCountOutEdge == 1) {
1115 uint64_t Total = 0;
1116 uint64_t OutSum = sumEdgeCount(Count->OutEdges);
1117 // If the one of the successor block can early terminate (no-return),
1118 // we can end up with situation where out edge sum count is larger as
1119 // the source BB's count is collected by a post-dominated block.
1120 if (Count->CountValue > OutSum)
1121 Total = Count->CountValue - OutSum;
1122 setEdgeCount(Count->OutEdges, Total);
1123 Changes = true;
1124 }
1125 if (Count->UnknownCountInEdge == 1) {
1126 uint64_t Total = 0;
1127 uint64_t InSum = sumEdgeCount(Count->InEdges);
1128 if (Count->CountValue > InSum)
1129 Total = Count->CountValue - InSum;
1130 setEdgeCount(Count->InEdges, Total);
1131 Changes = true;
1132 }
1133 }
1134 }
1135 }
1136
1137 LLVM_DEBUG(dbgs() << "Populate counts in " << NumPasses << " passes.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Populate counts in "
<< NumPasses << " passes.\n"; } } while (false)
;
1138#ifndef NDEBUG
1139 // Assert every BB has a valid counter.
1140 for (auto &BB : F) {
1141 auto BI = findBBInfo(&BB);
1142 if (BI == nullptr)
1143 continue;
1144 assert(BI->CountValid && "BB count is not valid")((BI->CountValid && "BB count is not valid") ? static_cast
<void> (0) : __assert_fail ("BI->CountValid && \"BB count is not valid\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1144, __PRETTY_FUNCTION__))
;
1145 }
1146#endif
1147 uint64_t FuncEntryCount = getBBInfo(&*F.begin()).CountValue;
1148 F.setEntryCount(ProfileCount(FuncEntryCount, Function::PCT_Real));
1149 uint64_t FuncMaxCount = FuncEntryCount;
1150 for (auto &BB : F) {
1151 auto BI = findBBInfo(&BB);
1152 if (BI == nullptr)
1153 continue;
1154 FuncMaxCount = std::max(FuncMaxCount, BI->CountValue);
1155 }
1156 markFunctionAttributes(FuncEntryCount, FuncMaxCount);
1157
1158 // Now annotate select instructions
1159 FuncInfo.SIVisitor.annotateSelects(F, this, &CountPosition);
1160 assert(CountPosition == ProfileCountSize)((CountPosition == ProfileCountSize) ? static_cast<void>
(0) : __assert_fail ("CountPosition == ProfileCountSize", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1160, __PRETTY_FUNCTION__))
;
1161
1162 LLVM_DEBUG(FuncInfo.dumpInfo("after reading profile."))do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { FuncInfo.dumpInfo("after reading profile."
); } } while (false)
;
1163}
1164
1165// Assign the scaled count values to the BB with multiple out edges.
1166void PGOUseFunc::setBranchWeights() {
1167 // Generate MD_prof metadata for every branch instruction.
1168 LLVM_DEBUG(dbgs() << "\nSetting branch weights.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "\nSetting branch weights.\n"
; } } while (false)
;
1169 for (auto &BB : F) {
1170 Instruction *TI = BB.getTerminator();
1171 if (TI->getNumSuccessors() < 2)
1172 continue;
1173 if (!(isa<BranchInst>(TI) || isa<SwitchInst>(TI) ||
1174 isa<IndirectBrInst>(TI)))
1175 continue;
1176 if (getBBInfo(&BB).CountValue == 0)
1177 continue;
1178
1179 // We have a non-zero Branch BB.
1180 const UseBBInfo &BBCountInfo = getBBInfo(&BB);
1181 unsigned Size = BBCountInfo.OutEdges.size();
1182 SmallVector<uint64_t, 2> EdgeCounts(Size, 0);
1183 uint64_t MaxCount = 0;
1184 for (unsigned s = 0; s < Size; s++) {
1185 const PGOUseEdge *E = BBCountInfo.OutEdges[s];
1186 const BasicBlock *SrcBB = E->SrcBB;
1187 const BasicBlock *DestBB = E->DestBB;
1188 if (DestBB == nullptr)
1189 continue;
1190 unsigned SuccNum = GetSuccessorNumber(SrcBB, DestBB);
1191 uint64_t EdgeCount = E->CountValue;
1192 if (EdgeCount > MaxCount)
1193 MaxCount = EdgeCount;
1194 EdgeCounts[SuccNum] = EdgeCount;
1195 }
1196 setProfMetadata(M, TI, EdgeCounts, MaxCount);
1197 }
1198}
1199
1200static bool isIndirectBrTarget(BasicBlock *BB) {
1201 for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
1202 if (isa<IndirectBrInst>((*PI)->getTerminator()))
1203 return true;
1204 }
1205 return false;
1206}
1207
1208void PGOUseFunc::annotateIrrLoopHeaderWeights() {
1209 LLVM_DEBUG(dbgs() << "\nAnnotating irreducible loop header weights.\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "\nAnnotating irreducible loop header weights.\n"
; } } while (false)
;
1210 // Find irr loop headers
1211 for (auto &BB : F) {
1212 // As a heuristic also annotate indrectbr targets as they have a high chance
1213 // to become an irreducible loop header after the indirectbr tail
1214 // duplication.
1215 if (BFI->isIrrLoopHeader(&BB) || isIndirectBrTarget(&BB)) {
1216 Instruction *TI = BB.getTerminator();
1217 const UseBBInfo &BBCountInfo = getBBInfo(&BB);
1218 setIrrLoopHeaderMetadata(M, TI, BBCountInfo.CountValue);
1219 }
1220 }
1221}
1222
1223void SelectInstVisitor::instrumentOneSelectInst(SelectInst &SI) {
1224 Module *M = F.getParent();
1225 IRBuilder<> Builder(&SI);
1226 Type *Int64Ty = Builder.getInt64Ty();
1227 Type *I8PtrTy = Builder.getInt8PtrTy();
1228 auto *Step = Builder.CreateZExt(SI.getCondition(), Int64Ty);
1229 Builder.CreateCall(
1230 Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment_step),
1231 {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
1232 Builder.getInt64(FuncHash), Builder.getInt32(TotalNumCtrs),
1233 Builder.getInt32(*CurCtrIdx), Step});
1234 ++(*CurCtrIdx);
1235}
1236
1237void SelectInstVisitor::annotateOneSelectInst(SelectInst &SI) {
1238 std::vector<uint64_t> &CountFromProfile = UseFunc->getProfileRecord().Counts;
1239 assert(*CurCtrIdx < CountFromProfile.size() &&((*CurCtrIdx < CountFromProfile.size() && "Out of bound access of counters"
) ? static_cast<void> (0) : __assert_fail ("*CurCtrIdx < CountFromProfile.size() && \"Out of bound access of counters\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1240, __PRETTY_FUNCTION__))
1240 "Out of bound access of counters")((*CurCtrIdx < CountFromProfile.size() && "Out of bound access of counters"
) ? static_cast<void> (0) : __assert_fail ("*CurCtrIdx < CountFromProfile.size() && \"Out of bound access of counters\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1240, __PRETTY_FUNCTION__))
;
1241 uint64_t SCounts[2];
1242 SCounts[0] = CountFromProfile[*CurCtrIdx]; // True count
1243 ++(*CurCtrIdx);
1244 uint64_t TotalCount = 0;
1245 auto BI = UseFunc->findBBInfo(SI.getParent());
1246 if (BI != nullptr)
1247 TotalCount = BI->CountValue;
1248 // False Count
1249 SCounts[1] = (TotalCount > SCounts[0] ? TotalCount - SCounts[0] : 0);
1250 uint64_t MaxCount = std::max(SCounts[0], SCounts[1]);
1251 if (MaxCount)
1252 setProfMetadata(F.getParent(), &SI, SCounts, MaxCount);
1253}
1254
1255void SelectInstVisitor::visitSelectInst(SelectInst &SI) {
1256 if (!PGOInstrSelect)
1257 return;
1258 // FIXME: do not handle this yet.
1259 if (SI.getCondition()->getType()->isVectorTy())
1260 return;
1261
1262 switch (Mode) {
1263 case VM_counting:
1264 NSIs++;
1265 return;
1266 case VM_instrument:
1267 instrumentOneSelectInst(SI);
1268 return;
1269 case VM_annotate:
1270 annotateOneSelectInst(SI);
1271 return;
1272 }
1273
1274 llvm_unreachable("Unknown visiting mode")::llvm::llvm_unreachable_internal("Unknown visiting mode", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1274)
;
1275}
1276
1277void MemIntrinsicVisitor::instrumentOneMemIntrinsic(MemIntrinsic &MI) {
1278 Module *M = F.getParent();
1279 IRBuilder<> Builder(&MI);
1280 Type *Int64Ty = Builder.getInt64Ty();
1281 Type *I8PtrTy = Builder.getInt8PtrTy();
1282 Value *Length = MI.getLength();
1283 assert(!dyn_cast<ConstantInt>(Length))((!dyn_cast<ConstantInt>(Length)) ? static_cast<void
> (0) : __assert_fail ("!dyn_cast<ConstantInt>(Length)"
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1283, __PRETTY_FUNCTION__))
;
1284 Builder.CreateCall(
1285 Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
1286 {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
1287 Builder.getInt64(FuncHash), Builder.CreateZExtOrTrunc(Length, Int64Ty),
1288 Builder.getInt32(IPVK_MemOPSize), Builder.getInt32(CurCtrId)});
1289 ++CurCtrId;
1290}
1291
1292void MemIntrinsicVisitor::visitMemIntrinsic(MemIntrinsic &MI) {
1293 if (!PGOInstrMemOP)
1294 return;
1295 Value *Length = MI.getLength();
1296 // Not instrument constant length calls.
1297 if (dyn_cast<ConstantInt>(Length))
1298 return;
1299
1300 switch (Mode) {
1301 case VM_counting:
1302 NMemIs++;
1303 return;
1304 case VM_instrument:
1305 instrumentOneMemIntrinsic(MI);
1306 return;
1307 case VM_annotate:
1308 Candidates.push_back(&MI);
1309 return;
1310 }
1311 llvm_unreachable("Unknown visiting mode")::llvm::llvm_unreachable_internal("Unknown visiting mode", "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1311)
;
1312}
1313
1314// Traverse all valuesites and annotate the instructions for all value kind.
1315void PGOUseFunc::annotateValueSites() {
1316 if (DisableValueProfiling)
1317 return;
1318
1319 // Create the PGOFuncName meta data.
1320 createPGOFuncNameMetadata(F, FuncInfo.FuncName);
1321
1322 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1323 annotateValueSites(Kind);
1324}
1325
1326// Annotate the instructions for a specific value kind.
1327void PGOUseFunc::annotateValueSites(uint32_t Kind) {
1328 unsigned ValueSiteIndex = 0;
1329 auto &ValueSites = FuncInfo.ValueSites[Kind];
1330 unsigned NumValueSites = ProfileRecord.getNumValueSites(Kind);
1331 if (NumValueSites != ValueSites.size()) {
1332 auto &Ctx = M->getContext();
1333 Ctx.diagnose(DiagnosticInfoPGOProfile(
1334 M->getName().data(),
1335 Twine("Inconsistent number of value sites for kind = ") + Twine(Kind) +
1336 " in " + F.getName().str(),
1337 DS_Warning));
1338 return;
1339 }
1340
1341 for (auto &I : ValueSites) {
1342 LLVM_DEBUG(dbgs() << "Read one value site profile (kind = " << Kinddo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Read one value site profile (kind = "
<< Kind << "): Index = " << ValueSiteIndex
<< " out of " << NumValueSites << "\n"; } }
while (false)
1343 << "): Index = " << ValueSiteIndex << " out of "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Read one value site profile (kind = "
<< Kind << "): Index = " << ValueSiteIndex
<< " out of " << NumValueSites << "\n"; } }
while (false)
1344 << NumValueSites << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Read one value site profile (kind = "
<< Kind << "): Index = " << ValueSiteIndex
<< " out of " << NumValueSites << "\n"; } }
while (false)
;
1345 annotateValueSite(*M, *I, ProfileRecord,
1346 static_cast<InstrProfValueKind>(Kind), ValueSiteIndex,
1347 Kind == IPVK_MemOPSize ? MaxNumMemOPAnnotations
1348 : MaxNumAnnotations);
1349 ValueSiteIndex++;
1350 }
1351}
1352
1353// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
1354// aware this is an ir_level profile so it can set the version flag.
1355static void createIRLevelProfileFlagVariable(Module &M) {
1356 Type *IntTy64 = Type::getInt64Ty(M.getContext());
1357 uint64_t ProfileVersion = (INSTR_PROF_RAW_VERSION4 | VARIANT_MASK_IR_PROF(0x1ULL << 56));
1358 auto IRLevelVersionVariable = new GlobalVariable(
1359 M, IntTy64, true, GlobalVariable::ExternalLinkage,
1360 Constant::getIntegerValue(IntTy64, APInt(64, ProfileVersion)),
1361 INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)"__llvm_profile_raw_version");
1362 IRLevelVersionVariable->setVisibility(GlobalValue::DefaultVisibility);
1363 Triple TT(M.getTargetTriple());
1364 if (!TT.supportsCOMDAT())
1365 IRLevelVersionVariable->setLinkage(GlobalValue::WeakAnyLinkage);
1366 else
1367 IRLevelVersionVariable->setComdat(M.getOrInsertComdat(
1368 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_RAW_VERSION_VAR)"__llvm_profile_raw_version")));
1369}
1370
1371// Collect the set of members for each Comdat in module M and store
1372// in ComdatMembers.
1373static void collectComdatMembers(
1374 Module &M,
1375 std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers) {
1376 if (!DoComdatRenaming)
1377 return;
1378 for (Function &F : M)
1379 if (Comdat *C = F.getComdat())
1380 ComdatMembers.insert(std::make_pair(C, &F));
1381 for (GlobalVariable &GV : M.globals())
1382 if (Comdat *C = GV.getComdat())
1383 ComdatMembers.insert(std::make_pair(C, &GV));
1384 for (GlobalAlias &GA : M.aliases())
1385 if (Comdat *C = GA.getComdat())
1386 ComdatMembers.insert(std::make_pair(C, &GA));
1387}
1388
1389static bool InstrumentAllFunctions(
1390 Module &M, function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
1391 function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) {
1392 createIRLevelProfileFlagVariable(M);
1393 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
1394 collectComdatMembers(M, ComdatMembers);
1395
1396 for (auto &F : M) {
1397 if (F.isDeclaration())
1398 continue;
1399 auto *BPI = LookupBPI(F);
1400 auto *BFI = LookupBFI(F);
1401 instrumentOneFunc(F, &M, BPI, BFI, ComdatMembers);
1402 }
1403 return true;
1404}
1405
1406bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) {
1407 if (skipModule(M))
1408 return false;
1409
1410 auto LookupBPI = [this](Function &F) {
1411 return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI();
1412 };
1413 auto LookupBFI = [this](Function &F) {
1414 return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
1415 };
1416 return InstrumentAllFunctions(M, LookupBPI, LookupBFI);
1417}
1418
1419PreservedAnalyses PGOInstrumentationGen::run(Module &M,
1420 ModuleAnalysisManager &AM) {
1421 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
1422 auto LookupBPI = [&FAM](Function &F) {
1423 return &FAM.getResult<BranchProbabilityAnalysis>(F);
1424 };
1425
1426 auto LookupBFI = [&FAM](Function &F) {
1427 return &FAM.getResult<BlockFrequencyAnalysis>(F);
1428 };
1429
1430 if (!InstrumentAllFunctions(M, LookupBPI, LookupBFI))
1431 return PreservedAnalyses::all();
1432
1433 return PreservedAnalyses::none();
1434}
1435
1436static bool annotateAllFunctions(
1437 Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName,
1438 function_ref<BranchProbabilityInfo *(Function &)> LookupBPI,
1439 function_ref<BlockFrequencyInfo *(Function &)> LookupBFI) {
1440 LLVM_DEBUG(dbgs() << "Read in profile counters: ")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Read in profile counters: "
; } } while (false)
;
1441 auto &Ctx = M.getContext();
1442 // Read the counter array from file.
1443 auto ReaderOrErr =
1444 IndexedInstrProfReader::create(ProfileFileName, ProfileRemappingFileName);
1445 if (Error E = ReaderOrErr.takeError()) {
4
Taking false branch
1446 handleAllErrors(std::move(E), [&](const ErrorInfoBase &EI) {
1447 Ctx.diagnose(
1448 DiagnosticInfoPGOProfile(ProfileFileName.data(), EI.message()));
1449 });
1450 return false;
1451 }
1452
1453 std::unique_ptr<IndexedInstrProfReader> PGOReader =
1454 std::move(ReaderOrErr.get());
1455 if (!PGOReader) {
5
Taking false branch
1456 Ctx.diagnose(DiagnosticInfoPGOProfile(ProfileFileName.data(),
1457 StringRef("Cannot get PGOReader")));
1458 return false;
1459 }
1460 // TODO: might need to change the warning once the clang option is finalized.
1461 if (!PGOReader->isIRLevelProfile()) {
6
Assuming the condition is false
7
Taking false branch
1462 Ctx.diagnose(DiagnosticInfoPGOProfile(
1463 ProfileFileName.data(), "Not an IR level instrumentation profile"));
1464 return false;
1465 }
1466
1467 std::unordered_multimap<Comdat *, GlobalValue *> ComdatMembers;
1468 collectComdatMembers(M, ComdatMembers);
1469 std::vector<Function *> HotFunctions;
1470 std::vector<Function *> ColdFunctions;
1471 for (auto &F : M) {
1472 if (F.isDeclaration())
8
Assuming the condition is false
9
Taking false branch
1473 continue;
1474 auto *BPI = LookupBPI(F);
1475 auto *BFI = LookupBFI(F);
1476 // Split indirectbr critical edges here before computing the MST rather than
1477 // later in getInstrBB() to avoid invalidating it.
1478 SplitIndirectBrCriticalEdges(F, BPI, BFI);
1479 PGOUseFunc Func(F, &M, ComdatMembers, BPI, BFI);
1480 if (!Func.readCounters(PGOReader.get()))
10
Calling 'PGOUseFunc::readCounters'
1481 continue;
1482 Func.populateCounters();
1483 Func.setBranchWeights();
1484 Func.annotateValueSites();
1485 Func.annotateIrrLoopHeaderWeights();
1486 PGOUseFunc::FuncFreqAttr FreqAttr = Func.getFuncFreqAttr();
1487 if (FreqAttr == PGOUseFunc::FFA_Cold)
1488 ColdFunctions.push_back(&F);
1489 else if (FreqAttr == PGOUseFunc::FFA_Hot)
1490 HotFunctions.push_back(&F);
1491 if (PGOViewCounts != PGOVCT_None &&
1492 (ViewBlockFreqFuncName.empty() ||
1493 F.getName().equals(ViewBlockFreqFuncName))) {
1494 LoopInfo LI{DominatorTree(F)};
1495 std::unique_ptr<BranchProbabilityInfo> NewBPI =
1496 llvm::make_unique<BranchProbabilityInfo>(F, LI);
1497 std::unique_ptr<BlockFrequencyInfo> NewBFI =
1498 llvm::make_unique<BlockFrequencyInfo>(F, *NewBPI, LI);
1499 if (PGOViewCounts == PGOVCT_Graph)
1500 NewBFI->view();
1501 else if (PGOViewCounts == PGOVCT_Text) {
1502 dbgs() << "pgo-view-counts: " << Func.getFunc().getName() << "\n";
1503 NewBFI->print(dbgs());
1504 }
1505 }
1506 if (PGOViewRawCounts != PGOVCT_None &&
1507 (ViewBlockFreqFuncName.empty() ||
1508 F.getName().equals(ViewBlockFreqFuncName))) {
1509 if (PGOViewRawCounts == PGOVCT_Graph)
1510 if (ViewBlockFreqFuncName.empty())
1511 WriteGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());
1512 else
1513 ViewGraph(&Func, Twine("PGORawCounts_") + Func.getFunc().getName());
1514 else if (PGOViewRawCounts == PGOVCT_Text) {
1515 dbgs() << "pgo-view-raw-counts: " << Func.getFunc().getName() << "\n";
1516 Func.dumpInfo();
1517 }
1518 }
1519 }
1520 M.setProfileSummary(PGOReader->getSummary().getMD(M.getContext()));
1521 // Set function hotness attribute from the profile.
1522 // We have to apply these attributes at the end because their presence
1523 // can affect the BranchProbabilityInfo of any callers, resulting in an
1524 // inconsistent MST between prof-gen and prof-use.
1525 for (auto &F : HotFunctions) {
1526 F->addFnAttr(Attribute::InlineHint);
1527 LLVM_DEBUG(dbgs() << "Set inline attribute to function: " << F->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Set inline attribute to function: "
<< F->getName() << "\n"; } } while (false)
1528 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Set inline attribute to function: "
<< F->getName() << "\n"; } } while (false)
;
1529 }
1530 for (auto &F : ColdFunctions) {
1531 F->addFnAttr(Attribute::Cold);
1532 LLVM_DEBUG(dbgs() << "Set cold attribute to function: " << F->getName()do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Set cold attribute to function: "
<< F->getName() << "\n"; } } while (false)
1533 << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Set cold attribute to function: "
<< F->getName() << "\n"; } } while (false)
;
1534 }
1535 return true;
1536}
1537
1538PGOInstrumentationUse::PGOInstrumentationUse(std::string Filename,
1539 std::string RemappingFilename)
1540 : ProfileFileName(std::move(Filename)),
1541 ProfileRemappingFileName(std::move(RemappingFilename)) {
1542 if (!PGOTestProfileFile.empty())
1543 ProfileFileName = PGOTestProfileFile;
1544 if (!PGOTestProfileRemappingFile.empty())
1545 ProfileRemappingFileName = PGOTestProfileRemappingFile;
1546}
1547
1548PreservedAnalyses PGOInstrumentationUse::run(Module &M,
1549 ModuleAnalysisManager &AM) {
1550
1551 auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
1552 auto LookupBPI = [&FAM](Function &F) {
1553 return &FAM.getResult<BranchProbabilityAnalysis>(F);
1554 };
1555
1556 auto LookupBFI = [&FAM](Function &F) {
1557 return &FAM.getResult<BlockFrequencyAnalysis>(F);
1558 };
1559
1560 if (!annotateAllFunctions(M, ProfileFileName, ProfileRemappingFileName,
1561 LookupBPI, LookupBFI))
1562 return PreservedAnalyses::all();
1563
1564 return PreservedAnalyses::none();
1565}
1566
1567bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) {
1568 if (skipModule(M))
1
Assuming the condition is false
2
Taking false branch
1569 return false;
1570
1571 auto LookupBPI = [this](Function &F) {
1572 return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(F).getBPI();
1573 };
1574 auto LookupBFI = [this](Function &F) {
1575 return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(F).getBFI();
1576 };
1577
1578 return annotateAllFunctions(M, ProfileFileName, "", LookupBPI, LookupBFI);
3
Calling 'annotateAllFunctions'
1579}
1580
1581static std::string getSimpleNodeName(const BasicBlock *Node) {
1582 if (!Node->getName().empty())
1583 return Node->getName();
1584
1585 std::string SimpleNodeName;
1586 raw_string_ostream OS(SimpleNodeName);
1587 Node->printAsOperand(OS, false);
1588 return OS.str();
1589}
1590
1591void llvm::setProfMetadata(Module *M, Instruction *TI,
1592 ArrayRef<uint64_t> EdgeCounts,
1593 uint64_t MaxCount) {
1594 MDBuilder MDB(M->getContext());
1595 assert(MaxCount > 0 && "Bad max count")((MaxCount > 0 && "Bad max count") ? static_cast<
void> (0) : __assert_fail ("MaxCount > 0 && \"Bad max count\""
, "/build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp"
, 1595, __PRETTY_FUNCTION__))
;
1596 uint64_t Scale = calculateCountScale(MaxCount);
1597 SmallVector<unsigned, 4> Weights;
1598 for (const auto &ECI : EdgeCounts)
1599 Weights.push_back(scaleBranchCount(ECI, Scale));
1600
1601 LLVM_DEBUG(dbgs() << "Weight is: "; for (const auto &Wdo { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Weight is: "; for
(const auto &W : Weights) { dbgs() << W << " "
; } dbgs() << "\n";; } } while (false)
1602 : Weights) {do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Weight is: "; for
(const auto &W : Weights) { dbgs() << W << " "
; } dbgs() << "\n";; } } while (false)
1603 dbgs() << W << " ";do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Weight is: "; for
(const auto &W : Weights) { dbgs() << W << " "
; } dbgs() << "\n";; } } while (false)
1604 } dbgs() << "\n";)do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("pgo-instrumentation")) { dbgs() << "Weight is: "; for
(const auto &W : Weights) { dbgs() << W << " "
; } dbgs() << "\n";; } } while (false)
;
1605 TI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
1606 if (EmitBranchProbability) {
1607 std::string BrCondStr = getBranchCondString(TI);
1608 if (BrCondStr.empty())
1609 return;
1610
1611 uint64_t WSum =
1612 std::accumulate(Weights.begin(), Weights.end(), (uint64_t)0,
1613 [](uint64_t w1, uint64_t w2) { return w1 + w2; });
1614 uint64_t TotalCount =
1615 std::accumulate(EdgeCounts.begin(), EdgeCounts.end(), (uint64_t)0,
1616 [](uint64_t c1, uint64_t c2) { return c1 + c2; });
1617 Scale = calculateCountScale(WSum);
1618 BranchProbability BP(scaleBranchCount(Weights[0], Scale),
1619 scaleBranchCount(WSum, Scale));
1620 std::string BranchProbStr;
1621 raw_string_ostream OS(BranchProbStr);
1622 OS << BP;
1623 OS << " (total count : " << TotalCount << ")";
1624 OS.flush();
1625 Function *F = TI->getParent()->getParent();
1626 OptimizationRemarkEmitter ORE(F);
1627 ORE.emit([&]() {
1628 return OptimizationRemark(DEBUG_TYPE"pgo-instrumentation", "pgo-instrumentation", TI)
1629 << BrCondStr << " is true with probability : " << BranchProbStr;
1630 });
1631 }
1632}
1633
1634namespace llvm {
1635
1636void setIrrLoopHeaderMetadata(Module *M, Instruction *TI, uint64_t Count) {
1637 MDBuilder MDB(M->getContext());
1638 TI->setMetadata(llvm::LLVMContext::MD_irr_loop,
1639 MDB.createIrrLoopHeaderWeight(Count));
1640}
1641
1642template <> struct GraphTraits<PGOUseFunc *> {
1643 using NodeRef = const BasicBlock *;
1644 using ChildIteratorType = succ_const_iterator;
1645 using nodes_iterator = pointer_iterator<Function::const_iterator>;
1646
1647 static NodeRef getEntryNode(const PGOUseFunc *G) {
1648 return &G->getFunc().front();
1649 }
1650
1651 static ChildIteratorType child_begin(const NodeRef N) {
1652 return succ_begin(N);
1653 }
1654
1655 static ChildIteratorType child_end(const NodeRef N) { return succ_end(N); }
1656
1657 static nodes_iterator nodes_begin(const PGOUseFunc *G) {
1658 return nodes_iterator(G->getFunc().begin());
1659 }
1660
1661 static nodes_iterator nodes_end(const PGOUseFunc *G) {
1662 return nodes_iterator(G->getFunc().end());
1663 }
1664};
1665
1666template <> struct DOTGraphTraits<PGOUseFunc *> : DefaultDOTGraphTraits {
1667 explicit DOTGraphTraits(bool isSimple = false)
1668 : DefaultDOTGraphTraits(isSimple) {}
1669
1670 static std::string getGraphName(const PGOUseFunc *G) {
1671 return G->getFunc().getName();
1672 }
1673
1674 std::string getNodeLabel(const BasicBlock *Node, const PGOUseFunc *Graph) {
1675 std::string Result;
1676 raw_string_ostream OS(Result);
1677
1678 OS << getSimpleNodeName(Node) << ":\\l";
1679 UseBBInfo *BI = Graph->findBBInfo(Node);
1680 OS << "Count : ";
1681 if (BI && BI->CountValid)
1682 OS << BI->CountValue << "\\l";
1683 else
1684 OS << "Unknown\\l";
1685
1686 if (!PGOInstrSelect)
1687 return Result;
1688
1689 for (auto BI = Node->begin(); BI != Node->end(); ++BI) {
1690 auto *I = &*BI;
1691 if (!isa<SelectInst>(I))
1692 continue;
1693 // Display scaled counts for SELECT instruction:
1694 OS << "SELECT : { T = ";
1695 uint64_t TC, FC;
1696 bool HasProf = I->extractProfMetadata(TC, FC);
1697 if (!HasProf)
1698 OS << "Unknown, F = Unknown }\\l";
1699 else
1700 OS << TC << ", F = " << FC << " }\\l";
1701 }
1702 return Result;
1703 }
1704};
1705
1706} // end namespace llvm

/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h

1//===- llvm/Support/Error.h - Recoverable error handling --------*- C++ -*-===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file defines an API used to report recoverable errors.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_SUPPORT_ERROR_H
15#define LLVM_SUPPORT_ERROR_H
16
17#include "llvm-c/Error.h"
18#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/SmallVector.h"
20#include "llvm/ADT/StringExtras.h"
21#include "llvm/ADT/Twine.h"
22#include "llvm/Config/abi-breaking.h"
23#include "llvm/Support/AlignOf.h"
24#include "llvm/Support/Compiler.h"
25#include "llvm/Support/Debug.h"
26#include "llvm/Support/ErrorHandling.h"
27#include "llvm/Support/ErrorOr.h"
28#include "llvm/Support/Format.h"
29#include "llvm/Support/raw_ostream.h"
30#include <algorithm>
31#include <cassert>
32#include <cstdint>
33#include <cstdlib>
34#include <functional>
35#include <memory>
36#include <new>
37#include <string>
38#include <system_error>
39#include <type_traits>
40#include <utility>
41#include <vector>
42
43namespace llvm {
44
45class ErrorSuccess;
46
47/// Base class for error info classes. Do not extend this directly: Extend
48/// the ErrorInfo template subclass instead.
49class ErrorInfoBase {
50public:
51 virtual ~ErrorInfoBase() = default;
52
53 /// Print an error message to an output stream.
54 virtual void log(raw_ostream &OS) const = 0;
55
56 /// Return the error message as a string.
57 virtual std::string message() const {
58 std::string Msg;
59 raw_string_ostream OS(Msg);
60 log(OS);
61 return OS.str();
62 }
63
64 /// Convert this error to a std::error_code.
65 ///
66 /// This is a temporary crutch to enable interaction with code still
67 /// using std::error_code. It will be removed in the future.
68 virtual std::error_code convertToErrorCode() const = 0;
69
70 // Returns the class ID for this type.
71 static const void *classID() { return &ID; }
72
73 // Returns the class ID for the dynamic type of this ErrorInfoBase instance.
74 virtual const void *dynamicClassID() const = 0;
75
76 // Check whether this instance is a subclass of the class identified by
77 // ClassID.
78 virtual bool isA(const void *const ClassID) const {
79 return ClassID == classID();
80 }
81
82 // Check whether this instance is a subclass of ErrorInfoT.
83 template <typename ErrorInfoT> bool isA() const {
84 return isA(ErrorInfoT::classID());
85 }
86
87private:
88 virtual void anchor();
89
90 static char ID;
91};
92
93/// Lightweight error class with error context and mandatory checking.
94///
95/// Instances of this class wrap a ErrorInfoBase pointer. Failure states
96/// are represented by setting the pointer to a ErrorInfoBase subclass
97/// instance containing information describing the failure. Success is
98/// represented by a null pointer value.
99///
100/// Instances of Error also contains a 'Checked' flag, which must be set
101/// before the destructor is called, otherwise the destructor will trigger a
102/// runtime error. This enforces at runtime the requirement that all Error
103/// instances be checked or returned to the caller.
104///
105/// There are two ways to set the checked flag, depending on what state the
106/// Error instance is in. For Error instances indicating success, it
107/// is sufficient to invoke the boolean conversion operator. E.g.:
108///
109/// @code{.cpp}
110/// Error foo(<...>);
111///
112/// if (auto E = foo(<...>))
113/// return E; // <- Return E if it is in the error state.
114/// // We have verified that E was in the success state. It can now be safely
115/// // destroyed.
116/// @endcode
117///
118/// A success value *can not* be dropped. For example, just calling 'foo(<...>)'
119/// without testing the return value will raise a runtime error, even if foo
120/// returns success.
121///
122/// For Error instances representing failure, you must use either the
123/// handleErrors or handleAllErrors function with a typed handler. E.g.:
124///
125/// @code{.cpp}
126/// class MyErrorInfo : public ErrorInfo<MyErrorInfo> {
127/// // Custom error info.
128/// };
129///
130/// Error foo(<...>) { return make_error<MyErrorInfo>(...); }
131///
132/// auto E = foo(<...>); // <- foo returns failure with MyErrorInfo.
133/// auto NewE =
134/// handleErrors(E,
135/// [](const MyErrorInfo &M) {
136/// // Deal with the error.
137/// },
138/// [](std::unique_ptr<OtherError> M) -> Error {
139/// if (canHandle(*M)) {
140/// // handle error.
141/// return Error::success();
142/// }
143/// // Couldn't handle this error instance. Pass it up the stack.
144/// return Error(std::move(M));
145/// );
146/// // Note - we must check or return NewE in case any of the handlers
147/// // returned a new error.
148/// @endcode
149///
150/// The handleAllErrors function is identical to handleErrors, except
151/// that it has a void return type, and requires all errors to be handled and
152/// no new errors be returned. It prevents errors (assuming they can all be
153/// handled) from having to be bubbled all the way to the top-level.
154///
155/// *All* Error instances must be checked before destruction, even if
156/// they're moved-assigned or constructed from Success values that have already
157/// been checked. This enforces checking through all levels of the call stack.
158class LLVM_NODISCARD[[clang::warn_unused_result]] Error {
159 // Both ErrorList and FileError need to be able to yank ErrorInfoBase
160 // pointers out of this class to add to the error list.
161 friend class ErrorList;
162 friend class FileError;
163
164 // handleErrors needs to be able to set the Checked flag.
165 template <typename... HandlerTs>
166 friend Error handleErrors(Error E, HandlerTs &&... Handlers);
167
168 // Expected<T> needs to be able to steal the payload when constructed from an
169 // error.
170 template <typename T> friend class Expected;
171
172 // wrap needs to be able to steal the payload.
173 friend LLVMErrorRef wrap(Error);
174
175protected:
176 /// Create a success value. Prefer using 'Error::success()' for readability
177 Error() {
178 setPtr(nullptr);
179 setChecked(false);
180 }
181
182public:
183 /// Create a success value.
184 static ErrorSuccess success();
185
186 // Errors are not copy-constructable.
187 Error(const Error &Other) = delete;
188
189 /// Move-construct an error value. The newly constructed error is considered
190 /// unchecked, even if the source error had been checked. The original error
191 /// becomes a checked Success value, regardless of its original state.
192 Error(Error &&Other) {
193 setChecked(true);
194 *this = std::move(Other);
195 }
196
197 /// Create an error value. Prefer using the 'make_error' function, but
198 /// this constructor can be useful when "re-throwing" errors from handlers.
199 Error(std::unique_ptr<ErrorInfoBase> Payload) {
200 setPtr(Payload.release());
201 setChecked(false);
26
Potential leak of memory pointed to by 'Payload._M_t._M_head_impl'
202 }
203
204 // Errors are not copy-assignable.
205 Error &operator=(const Error &Other) = delete;
206
207 /// Move-assign an error value. The current error must represent success, you
208 /// you cannot overwrite an unhandled error. The current error is then
209 /// considered unchecked. The source error becomes a checked success value,
210 /// regardless of its original state.
211 Error &operator=(Error &&Other) {
212 // Don't allow overwriting of unchecked values.
213 assertIsChecked();
214 setPtr(Other.getPtr());
215
216 // This Error is unchecked, even if the source error was checked.
217 setChecked(false);
218
219 // Null out Other's payload and set its checked bit.
220 Other.setPtr(nullptr);
221 Other.setChecked(true);
222
223 return *this;
224 }
225
226 /// Destroy a Error. Fails with a call to abort() if the error is
227 /// unchecked.
228 ~Error() {
229 assertIsChecked();
230 delete getPtr();
231 }
232
233 /// Bool conversion. Returns true if this Error is in a failure state,
234 /// and false if it is in an accept state. If the error is in a Success state
235 /// it will be considered checked.
236 explicit operator bool() {
237 setChecked(getPtr() == nullptr);
238 return getPtr() != nullptr;
239 }
240
241 /// Check whether one error is a subclass of another.
242 template <typename ErrT> bool isA() const {
243 return getPtr() && getPtr()->isA(ErrT::classID());
244 }
245
246 /// Returns the dynamic class id of this error, or null if this is a success
247 /// value.
248 const void* dynamicClassID() const {
249 if (!getPtr())
250 return nullptr;
251 return getPtr()->dynamicClassID();
252 }
253
254private:
255#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
256 // assertIsChecked() happens very frequently, but under normal circumstances
257 // is supposed to be a no-op. So we want it to be inlined, but having a bunch
258 // of debug prints can cause the function to be too large for inlining. So
259 // it's important that we define this function out of line so that it can't be
260 // inlined.
261 LLVM_ATTRIBUTE_NORETURN__attribute__((noreturn))
262 void fatalUncheckedError() const;
263#endif
264
265 void assertIsChecked() {
266#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
267 if (LLVM_UNLIKELY(!getChecked() || getPtr())__builtin_expect((bool)(!getChecked() || getPtr()), false))
268 fatalUncheckedError();
269#endif
270 }
271
272 ErrorInfoBase *getPtr() const {
273 return reinterpret_cast<ErrorInfoBase*>(
274 reinterpret_cast<uintptr_t>(Payload) &
275 ~static_cast<uintptr_t>(0x1));
276 }
277
278 void setPtr(ErrorInfoBase *EI) {
279#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
280 Payload = reinterpret_cast<ErrorInfoBase*>(
281 (reinterpret_cast<uintptr_t>(EI) &
282 ~static_cast<uintptr_t>(0x1)) |
283 (reinterpret_cast<uintptr_t>(Payload) & 0x1));
284#else
285 Payload = EI;
286#endif
287 }
288
289 bool getChecked() const {
290#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
291 return (reinterpret_cast<uintptr_t>(Payload) & 0x1) == 0;
292#else
293 return true;
294#endif
295 }
296
297 void setChecked(bool V) {
298 Payload = reinterpret_cast<ErrorInfoBase*>(
299 (reinterpret_cast<uintptr_t>(Payload) &
300 ~static_cast<uintptr_t>(0x1)) |
301 (V ? 0 : 1));
302 }
303
304 std::unique_ptr<ErrorInfoBase> takePayload() {
305 std::unique_ptr<ErrorInfoBase> Tmp(getPtr());
306 setPtr(nullptr);
307 setChecked(true);
308 return Tmp;
309 }
310
311 friend raw_ostream &operator<<(raw_ostream &OS, const Error &E) {
312 if (auto P = E.getPtr())
313 P->log(OS);
314 else
315 OS << "success";
316 return OS;
317 }
318
319 ErrorInfoBase *Payload = nullptr;
320};
321
322/// Subclass of Error for the sole purpose of identifying the success path in
323/// the type system. This allows to catch invalid conversion to Expected<T> at
324/// compile time.
325class ErrorSuccess final : public Error {};
326
327inline ErrorSuccess Error::success() { return ErrorSuccess(); }
328
329/// Make a Error instance representing failure using the given error info
330/// type.
331template <typename ErrT, typename... ArgTs> Error make_error(ArgTs &&... Args) {
332 return Error(llvm::make_unique<ErrT>(std::forward<ArgTs>(Args)...));
333}
334
335/// Base class for user error types. Users should declare their error types
336/// like:
337///
338/// class MyError : public ErrorInfo<MyError> {
339/// ....
340/// };
341///
342/// This class provides an implementation of the ErrorInfoBase::kind
343/// method, which is used by the Error RTTI system.
344template <typename ThisErrT, typename ParentErrT = ErrorInfoBase>
345class ErrorInfo : public ParentErrT {
346public:
347 using ParentErrT::ParentErrT; // inherit constructors
348
349 static const void *classID() { return &ThisErrT::ID; }
350
351 const void *dynamicClassID() const override { return &ThisErrT::ID; }
352
353 bool isA(const void *const ClassID) const override {
354 return ClassID == classID() || ParentErrT::isA(ClassID);
355 }
356};
357
358/// Special ErrorInfo subclass representing a list of ErrorInfos.
359/// Instances of this class are constructed by joinError.
360class ErrorList final : public ErrorInfo<ErrorList> {
361 // handleErrors needs to be able to iterate the payload list of an
362 // ErrorList.
363 template <typename... HandlerTs>
364 friend Error handleErrors(Error E, HandlerTs &&... Handlers);
365
366 // joinErrors is implemented in terms of join.
367 friend Error joinErrors(Error, Error);
368
369public:
370 void log(raw_ostream &OS) const override {
371 OS << "Multiple errors:\n";
372 for (auto &ErrPayload : Payloads) {
373 ErrPayload->log(OS);
374 OS << "\n";
375 }
376 }
377
378 std::error_code convertToErrorCode() const override;
379
380 // Used by ErrorInfo::classID.
381 static char ID;
382
383private:
384 ErrorList(std::unique_ptr<ErrorInfoBase> Payload1,
385 std::unique_ptr<ErrorInfoBase> Payload2) {
386 assert(!Payload1->isA<ErrorList>() && !Payload2->isA<ErrorList>() &&((!Payload1->isA<ErrorList>() && !Payload2->
isA<ErrorList>() && "ErrorList constructor payloads should be singleton errors"
) ? static_cast<void> (0) : __assert_fail ("!Payload1->isA<ErrorList>() && !Payload2->isA<ErrorList>() && \"ErrorList constructor payloads should be singleton errors\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 387, __PRETTY_FUNCTION__))
387 "ErrorList constructor payloads should be singleton errors")((!Payload1->isA<ErrorList>() && !Payload2->
isA<ErrorList>() && "ErrorList constructor payloads should be singleton errors"
) ? static_cast<void> (0) : __assert_fail ("!Payload1->isA<ErrorList>() && !Payload2->isA<ErrorList>() && \"ErrorList constructor payloads should be singleton errors\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 387, __PRETTY_FUNCTION__))
;
388 Payloads.push_back(std::move(Payload1));
389 Payloads.push_back(std::move(Payload2));
390 }
391
392 static Error join(Error E1, Error E2) {
393 if (!E1)
18
Taking false branch
394 return E2;
395 if (!E2)
19
Taking false branch
396 return E1;
397 if (E1.isA<ErrorList>()) {
20
Assuming the condition is false
21
Taking false branch
398 auto &E1List = static_cast<ErrorList &>(*E1.getPtr());
399 if (E2.isA<ErrorList>()) {
400 auto E2Payload = E2.takePayload();
401 auto &E2List = static_cast<ErrorList &>(*E2Payload);
402 for (auto &Payload : E2List.Payloads)
403 E1List.Payloads.push_back(std::move(Payload));
404 } else
405 E1List.Payloads.push_back(E2.takePayload());
406
407 return E1;
408 }
409 if (E2.isA<ErrorList>()) {
22
Assuming the condition is false
23
Taking false branch
410 auto &E2List = static_cast<ErrorList &>(*E2.getPtr());
411 E2List.Payloads.insert(E2List.Payloads.begin(), E1.takePayload());
412 return E2;
413 }
414 return Error(std::unique_ptr<ErrorList>(
25
Calling constructor for 'Error'
415 new ErrorList(E1.takePayload(), E2.takePayload())));
24
Memory is allocated
416 }
417
418 std::vector<std::unique_ptr<ErrorInfoBase>> Payloads;
419};
420
421/// Concatenate errors. The resulting Error is unchecked, and contains the
422/// ErrorInfo(s), if any, contained in E1, followed by the
423/// ErrorInfo(s), if any, contained in E2.
424inline Error joinErrors(Error E1, Error E2) {
425 return ErrorList::join(std::move(E1), std::move(E2));
426}
427
428/// Tagged union holding either a T or a Error.
429///
430/// This class parallels ErrorOr, but replaces error_code with Error. Since
431/// Error cannot be copied, this class replaces getError() with
432/// takeError(). It also adds an bool errorIsA<ErrT>() method for testing the
433/// error class type.
434template <class T> class LLVM_NODISCARD[[clang::warn_unused_result]] Expected {
435 template <class T1> friend class ExpectedAsOutParameter;
436 template <class OtherT> friend class Expected;
437
438 static const bool isRef = std::is_reference<T>::value;
439
440 using wrap = std::reference_wrapper<typename std::remove_reference<T>::type>;
441
442 using error_type = std::unique_ptr<ErrorInfoBase>;
443
444public:
445 using storage_type = typename std::conditional<isRef, wrap, T>::type;
446 using value_type = T;
447
448private:
449 using reference = typename std::remove_reference<T>::type &;
450 using const_reference = const typename std::remove_reference<T>::type &;
451 using pointer = typename std::remove_reference<T>::type *;
452 using const_pointer = const typename std::remove_reference<T>::type *;
453
454public:
455 /// Create an Expected<T> error value from the given Error.
456 Expected(Error Err)
457 : HasError(true)
458#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
459 // Expected is unchecked upon construction in Debug builds.
460 , Unchecked(true)
461#endif
462 {
463 assert(Err && "Cannot create Expected<T> from Error success value.")((Err && "Cannot create Expected<T> from Error success value."
) ? static_cast<void> (0) : __assert_fail ("Err && \"Cannot create Expected<T> from Error success value.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 463, __PRETTY_FUNCTION__))
;
464 new (getErrorStorage()) error_type(Err.takePayload());
465 }
466
467 /// Forbid to convert from Error::success() implicitly, this avoids having
468 /// Expected<T> foo() { return Error::success(); } which compiles otherwise
469 /// but triggers the assertion above.
470 Expected(ErrorSuccess) = delete;
471
472 /// Create an Expected<T> success value from the given OtherT value, which
473 /// must be convertible to T.
474 template <typename OtherT>
475 Expected(OtherT &&Val,
476 typename std::enable_if<std::is_convertible<OtherT, T>::value>::type
477 * = nullptr)
478 : HasError(false)
479#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
480 // Expected is unchecked upon construction in Debug builds.
481 , Unchecked(true)
482#endif
483 {
484 new (getStorage()) storage_type(std::forward<OtherT>(Val));
485 }
486
487 /// Move construct an Expected<T> value.
488 Expected(Expected &&Other) { moveConstruct(std::move(Other)); }
489
490 /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT
491 /// must be convertible to T.
492 template <class OtherT>
493 Expected(Expected<OtherT> &&Other,
494 typename std::enable_if<std::is_convertible<OtherT, T>::value>::type
495 * = nullptr) {
496 moveConstruct(std::move(Other));
497 }
498
499 /// Move construct an Expected<T> value from an Expected<OtherT>, where OtherT
500 /// isn't convertible to T.
501 template <class OtherT>
502 explicit Expected(
503 Expected<OtherT> &&Other,
504 typename std::enable_if<!std::is_convertible<OtherT, T>::value>::type * =
505 nullptr) {
506 moveConstruct(std::move(Other));
507 }
508
509 /// Move-assign from another Expected<T>.
510 Expected &operator=(Expected &&Other) {
511 moveAssign(std::move(Other));
512 return *this;
513 }
514
515 /// Destroy an Expected<T>.
516 ~Expected() {
517 assertIsChecked();
518 if (!HasError)
519 getStorage()->~storage_type();
520 else
521 getErrorStorage()->~error_type();
522 }
523
524 /// Return false if there is an error.
525 explicit operator bool() {
526#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
527 Unchecked = HasError;
528#endif
529 return !HasError;
530 }
531
532 /// Returns a reference to the stored T value.
533 reference get() {
534 assertIsChecked();
535 return *getStorage();
536 }
537
538 /// Returns a const reference to the stored T value.
539 const_reference get() const {
540 assertIsChecked();
541 return const_cast<Expected<T> *>(this)->get();
542 }
543
544 /// Check that this Expected<T> is an error of type ErrT.
545 template <typename ErrT> bool errorIsA() const {
546 return HasError && (*getErrorStorage())->template isA<ErrT>();
547 }
548
549 /// Take ownership of the stored error.
550 /// After calling this the Expected<T> is in an indeterminate state that can
551 /// only be safely destructed. No further calls (beside the destructor) should
552 /// be made on the Expected<T> vaule.
553 Error takeError() {
554#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
555 Unchecked = false;
556#endif
557 return HasError ? Error(std::move(*getErrorStorage())) : Error::success();
558 }
559
560 /// Returns a pointer to the stored T value.
561 pointer operator->() {
562 assertIsChecked();
563 return toPointer(getStorage());
564 }
565
566 /// Returns a const pointer to the stored T value.
567 const_pointer operator->() const {
568 assertIsChecked();
569 return toPointer(getStorage());
570 }
571
572 /// Returns a reference to the stored T value.
573 reference operator*() {
574 assertIsChecked();
575 return *getStorage();
576 }
577
578 /// Returns a const reference to the stored T value.
579 const_reference operator*() const {
580 assertIsChecked();
581 return *getStorage();
582 }
583
584private:
585 template <class T1>
586 static bool compareThisIfSameType(const T1 &a, const T1 &b) {
587 return &a == &b;
588 }
589
590 template <class T1, class T2>
591 static bool compareThisIfSameType(const T1 &a, const T2 &b) {
592 return false;
593 }
594
595 template <class OtherT> void moveConstruct(Expected<OtherT> &&Other) {
596 HasError = Other.HasError;
597#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
598 Unchecked = true;
599 Other.Unchecked = false;
600#endif
601
602 if (!HasError)
603 new (getStorage()) storage_type(std::move(*Other.getStorage()));
604 else
605 new (getErrorStorage()) error_type(std::move(*Other.getErrorStorage()));
606 }
607
608 template <class OtherT> void moveAssign(Expected<OtherT> &&Other) {
609 assertIsChecked();
610
611 if (compareThisIfSameType(*this, Other))
612 return;
613
614 this->~Expected();
615 new (this) Expected(std::move(Other));
616 }
617
618 pointer toPointer(pointer Val) { return Val; }
619
620 const_pointer toPointer(const_pointer Val) const { return Val; }
621
622 pointer toPointer(wrap *Val) { return &Val->get(); }
623
624 const_pointer toPointer(const wrap *Val) const { return &Val->get(); }
625
626 storage_type *getStorage() {
627 assert(!HasError && "Cannot get value when an error exists!")((!HasError && "Cannot get value when an error exists!"
) ? static_cast<void> (0) : __assert_fail ("!HasError && \"Cannot get value when an error exists!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 627, __PRETTY_FUNCTION__))
;
628 return reinterpret_cast<storage_type *>(TStorage.buffer);
629 }
630
631 const storage_type *getStorage() const {
632 assert(!HasError && "Cannot get value when an error exists!")((!HasError && "Cannot get value when an error exists!"
) ? static_cast<void> (0) : __assert_fail ("!HasError && \"Cannot get value when an error exists!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 632, __PRETTY_FUNCTION__))
;
633 return reinterpret_cast<const storage_type *>(TStorage.buffer);
634 }
635
636 error_type *getErrorStorage() {
637 assert(HasError && "Cannot get error when a value exists!")((HasError && "Cannot get error when a value exists!"
) ? static_cast<void> (0) : __assert_fail ("HasError && \"Cannot get error when a value exists!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 637, __PRETTY_FUNCTION__))
;
638 return reinterpret_cast<error_type *>(ErrorStorage.buffer);
639 }
640
641 const error_type *getErrorStorage() const {
642 assert(HasError && "Cannot get error when a value exists!")((HasError && "Cannot get error when a value exists!"
) ? static_cast<void> (0) : __assert_fail ("HasError && \"Cannot get error when a value exists!\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 642, __PRETTY_FUNCTION__))
;
643 return reinterpret_cast<const error_type *>(ErrorStorage.buffer);
644 }
645
646 // Used by ExpectedAsOutParameter to reset the checked flag.
647 void setUnchecked() {
648#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
649 Unchecked = true;
650#endif
651 }
652
653#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
654 LLVM_ATTRIBUTE_NORETURN__attribute__((noreturn))
655 LLVM_ATTRIBUTE_NOINLINE__attribute__((noinline))
656 void fatalUncheckedExpected() const {
657 dbgs() << "Expected<T> must be checked before access or destruction.\n";
658 if (HasError) {
659 dbgs() << "Unchecked Expected<T> contained error:\n";
660 (*getErrorStorage())->log(dbgs());
661 } else
662 dbgs() << "Expected<T> value was in success state. (Note: Expected<T> "
663 "values in success mode must still be checked prior to being "
664 "destroyed).\n";
665 abort();
666 }
667#endif
668
669 void assertIsChecked() {
670#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
671 if (LLVM_UNLIKELY(Unchecked)__builtin_expect((bool)(Unchecked), false))
672 fatalUncheckedExpected();
673#endif
674 }
675
676 union {
677 AlignedCharArrayUnion<storage_type> TStorage;
678 AlignedCharArrayUnion<error_type> ErrorStorage;
679 };
680 bool HasError : 1;
681#if LLVM_ENABLE_ABI_BREAKING_CHECKS1
682 bool Unchecked : 1;
683#endif
684};
685
686/// Report a serious error, calling any installed error handler. See
687/// ErrorHandling.h.
688LLVM_ATTRIBUTE_NORETURN__attribute__((noreturn)) void report_fatal_error(Error Err,
689 bool gen_crash_diag = true);
690
691/// Report a fatal error if Err is a failure value.
692///
693/// This function can be used to wrap calls to fallible functions ONLY when it
694/// is known that the Error will always be a success value. E.g.
695///
696/// @code{.cpp}
697/// // foo only attempts the fallible operation if DoFallibleOperation is
698/// // true. If DoFallibleOperation is false then foo always returns
699/// // Error::success().
700/// Error foo(bool DoFallibleOperation);
701///
702/// cantFail(foo(false));
703/// @endcode
704inline void cantFail(Error Err, const char *Msg = nullptr) {
705 if (Err) {
706 if (!Msg)
707 Msg = "Failure value returned from cantFail wrapped call";
708 llvm_unreachable(Msg)::llvm::llvm_unreachable_internal(Msg, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 708)
;
709 }
710}
711
712/// Report a fatal error if ValOrErr is a failure value, otherwise unwraps and
713/// returns the contained value.
714///
715/// This function can be used to wrap calls to fallible functions ONLY when it
716/// is known that the Error will always be a success value. E.g.
717///
718/// @code{.cpp}
719/// // foo only attempts the fallible operation if DoFallibleOperation is
720/// // true. If DoFallibleOperation is false then foo always returns an int.
721/// Expected<int> foo(bool DoFallibleOperation);
722///
723/// int X = cantFail(foo(false));
724/// @endcode
725template <typename T>
726T cantFail(Expected<T> ValOrErr, const char *Msg = nullptr) {
727 if (ValOrErr)
728 return std::move(*ValOrErr);
729 else {
730 if (!Msg)
731 Msg = "Failure value returned from cantFail wrapped call";
732 llvm_unreachable(Msg)::llvm::llvm_unreachable_internal(Msg, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 732)
;
733 }
734}
735
736/// Report a fatal error if ValOrErr is a failure value, otherwise unwraps and
737/// returns the contained reference.
738///
739/// This function can be used to wrap calls to fallible functions ONLY when it
740/// is known that the Error will always be a success value. E.g.
741///
742/// @code{.cpp}
743/// // foo only attempts the fallible operation if DoFallibleOperation is
744/// // true. If DoFallibleOperation is false then foo always returns a Bar&.
745/// Expected<Bar&> foo(bool DoFallibleOperation);
746///
747/// Bar &X = cantFail(foo(false));
748/// @endcode
749template <typename T>
750T& cantFail(Expected<T&> ValOrErr, const char *Msg = nullptr) {
751 if (ValOrErr)
752 return *ValOrErr;
753 else {
754 if (!Msg)
755 Msg = "Failure value returned from cantFail wrapped call";
756 llvm_unreachable(Msg)::llvm::llvm_unreachable_internal(Msg, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 756)
;
757 }
758}
759
760/// Helper for testing applicability of, and applying, handlers for
761/// ErrorInfo types.
762template <typename HandlerT>
763class ErrorHandlerTraits
764 : public ErrorHandlerTraits<decltype(
765 &std::remove_reference<HandlerT>::type::operator())> {};
766
767// Specialization functions of the form 'Error (const ErrT&)'.
768template <typename ErrT> class ErrorHandlerTraits<Error (&)(ErrT &)> {
769public:
770 static bool appliesTo(const ErrorInfoBase &E) {
771 return E.template isA<ErrT>();
772 }
773
774 template <typename HandlerT>
775 static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
776 assert(appliesTo(*E) && "Applying incorrect handler")((appliesTo(*E) && "Applying incorrect handler") ? static_cast
<void> (0) : __assert_fail ("appliesTo(*E) && \"Applying incorrect handler\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 776, __PRETTY_FUNCTION__))
;
777 return H(static_cast<ErrT &>(*E));
778 }
779};
780
781// Specialization functions of the form 'void (const ErrT&)'.
782template <typename ErrT> class ErrorHandlerTraits<void (&)(ErrT &)> {
783public:
784 static bool appliesTo(const ErrorInfoBase &E) {
785 return E.template isA<ErrT>();
786 }
787
788 template <typename HandlerT>
789 static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
790 assert(appliesTo(*E) && "Applying incorrect handler")((appliesTo(*E) && "Applying incorrect handler") ? static_cast
<void> (0) : __assert_fail ("appliesTo(*E) && \"Applying incorrect handler\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 790, __PRETTY_FUNCTION__))
;
791 H(static_cast<ErrT &>(*E));
792 return Error::success();
793 }
794};
795
796/// Specialization for functions of the form 'Error (std::unique_ptr<ErrT>)'.
797template <typename ErrT>
798class ErrorHandlerTraits<Error (&)(std::unique_ptr<ErrT>)> {
799public:
800 static bool appliesTo(const ErrorInfoBase &E) {
801 return E.template isA<ErrT>();
802 }
803
804 template <typename HandlerT>
805 static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
806 assert(appliesTo(*E) && "Applying incorrect handler")((appliesTo(*E) && "Applying incorrect handler") ? static_cast
<void> (0) : __assert_fail ("appliesTo(*E) && \"Applying incorrect handler\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 806, __PRETTY_FUNCTION__))
;
807 std::unique_ptr<ErrT> SubE(static_cast<ErrT *>(E.release()));
808 return H(std::move(SubE));
809 }
810};
811
812/// Specialization for functions of the form 'void (std::unique_ptr<ErrT>)'.
813template <typename ErrT>
814class ErrorHandlerTraits<void (&)(std::unique_ptr<ErrT>)> {
815public:
816 static bool appliesTo(const ErrorInfoBase &E) {
817 return E.template isA<ErrT>();
818 }
819
820 template <typename HandlerT>
821 static Error apply(HandlerT &&H, std::unique_ptr<ErrorInfoBase> E) {
822 assert(appliesTo(*E) && "Applying incorrect handler")((appliesTo(*E) && "Applying incorrect handler") ? static_cast
<void> (0) : __assert_fail ("appliesTo(*E) && \"Applying incorrect handler\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 822, __PRETTY_FUNCTION__))
;
823 std::unique_ptr<ErrT> SubE(static_cast<ErrT *>(E.release()));
824 H(std::move(SubE));
825 return Error::success();
826 }
827};
828
829// Specialization for member functions of the form 'RetT (const ErrT&)'.
830template <typename C, typename RetT, typename ErrT>
831class ErrorHandlerTraits<RetT (C::*)(ErrT &)>
832 : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
833
834// Specialization for member functions of the form 'RetT (const ErrT&) const'.
835template <typename C, typename RetT, typename ErrT>
836class ErrorHandlerTraits<RetT (C::*)(ErrT &) const>
837 : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
838
839// Specialization for member functions of the form 'RetT (const ErrT&)'.
840template <typename C, typename RetT, typename ErrT>
841class ErrorHandlerTraits<RetT (C::*)(const ErrT &)>
842 : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
843
844// Specialization for member functions of the form 'RetT (const ErrT&) const'.
845template <typename C, typename RetT, typename ErrT>
846class ErrorHandlerTraits<RetT (C::*)(const ErrT &) const>
847 : public ErrorHandlerTraits<RetT (&)(ErrT &)> {};
848
849/// Specialization for member functions of the form
850/// 'RetT (std::unique_ptr<ErrT>)'.
851template <typename C, typename RetT, typename ErrT>
852class ErrorHandlerTraits<RetT (C::*)(std::unique_ptr<ErrT>)>
853 : public ErrorHandlerTraits<RetT (&)(std::unique_ptr<ErrT>)> {};
854
855/// Specialization for member functions of the form
856/// 'RetT (std::unique_ptr<ErrT>) const'.
857template <typename C, typename RetT, typename ErrT>
858class ErrorHandlerTraits<RetT (C::*)(std::unique_ptr<ErrT>) const>
859 : public ErrorHandlerTraits<RetT (&)(std::unique_ptr<ErrT>)> {};
860
861inline Error handleErrorImpl(std::unique_ptr<ErrorInfoBase> Payload) {
862 return Error(std::move(Payload));
863}
864
865template <typename HandlerT, typename... HandlerTs>
866Error handleErrorImpl(std::unique_ptr<ErrorInfoBase> Payload,
867 HandlerT &&Handler, HandlerTs &&... Handlers) {
868 if (ErrorHandlerTraits<HandlerT>::appliesTo(*Payload))
869 return ErrorHandlerTraits<HandlerT>::apply(std::forward<HandlerT>(Handler),
870 std::move(Payload));
871 return handleErrorImpl(std::move(Payload),
872 std::forward<HandlerTs>(Handlers)...);
873}
874
875/// Pass the ErrorInfo(s) contained in E to their respective handlers. Any
876/// unhandled errors (or Errors returned by handlers) are re-concatenated and
877/// returned.
878/// Because this function returns an error, its result must also be checked
879/// or returned. If you intend to handle all errors use handleAllErrors
880/// (which returns void, and will abort() on unhandled errors) instead.
881template <typename... HandlerTs>
882Error handleErrors(Error E, HandlerTs &&... Hs) {
883 if (!E)
14
Taking false branch
884 return Error::success();
885
886 std::unique_ptr<ErrorInfoBase> Payload = E.takePayload();
887
888 if (Payload->isA<ErrorList>()) {
15
Assuming the condition is true
16
Taking true branch
889 ErrorList &List = static_cast<ErrorList &>(*Payload);
890 Error R;
891 for (auto &P : List.Payloads)
892 R = ErrorList::join(
17
Calling 'ErrorList::join'
893 std::move(R),
894 handleErrorImpl(std::move(P), std::forward<HandlerTs>(Hs)...));
895 return R;
896 }
897
898 return handleErrorImpl(std::move(Payload), std::forward<HandlerTs>(Hs)...);
899}
900
901/// Behaves the same as handleErrors, except that by contract all errors
902/// *must* be handled by the given handlers (i.e. there must be no remaining
903/// errors after running the handlers, or llvm_unreachable is called).
904template <typename... HandlerTs>
905void handleAllErrors(Error E, HandlerTs &&... Handlers) {
906 cantFail(handleErrors(std::move(E), std::forward<HandlerTs>(Handlers)...));
13
Calling 'handleErrors<(lambda at /build/llvm-toolchain-snapshot-8~svn345461/lib/Transforms/Instrumentation/PGOInstrumentation.cpp:1021:35)>'
907}
908
909/// Check that E is a non-error, then drop it.
910/// If E is an error, llvm_unreachable will be called.
911inline void handleAllErrors(Error E) {
912 cantFail(std::move(E));
913}
914
915/// Handle any errors (if present) in an Expected<T>, then try a recovery path.
916///
917/// If the incoming value is a success value it is returned unmodified. If it
918/// is a failure value then it the contained error is passed to handleErrors.
919/// If handleErrors is able to handle the error then the RecoveryPath functor
920/// is called to supply the final result. If handleErrors is not able to
921/// handle all errors then the unhandled errors are returned.
922///
923/// This utility enables the follow pattern:
924///
925/// @code{.cpp}
926/// enum FooStrategy { Aggressive, Conservative };
927/// Expected<Foo> foo(FooStrategy S);
928///
929/// auto ResultOrErr =
930/// handleExpected(
931/// foo(Aggressive),
932/// []() { return foo(Conservative); },
933/// [](AggressiveStrategyError&) {
934/// // Implicitly conusme this - we'll recover by using a conservative
935/// // strategy.
936/// });
937///
938/// @endcode
939template <typename T, typename RecoveryFtor, typename... HandlerTs>
940Expected<T> handleExpected(Expected<T> ValOrErr, RecoveryFtor &&RecoveryPath,
941 HandlerTs &&... Handlers) {
942 if (ValOrErr)
943 return ValOrErr;
944
945 if (auto Err = handleErrors(ValOrErr.takeError(),
946 std::forward<HandlerTs>(Handlers)...))
947 return std::move(Err);
948
949 return RecoveryPath();
950}
951
952/// Log all errors (if any) in E to OS. If there are any errors, ErrorBanner
953/// will be printed before the first one is logged. A newline will be printed
954/// after each error.
955///
956/// This is useful in the base level of your program to allow clean termination
957/// (allowing clean deallocation of resources, etc.), while reporting error
958/// information to the user.
959void logAllUnhandledErrors(Error E, raw_ostream &OS, Twine ErrorBanner);
960
961/// Write all error messages (if any) in E to a string. The newline character
962/// is used to separate error messages.
963inline std::string toString(Error E) {
964 SmallVector<std::string, 2> Errors;
965 handleAllErrors(std::move(E), [&Errors](const ErrorInfoBase &EI) {
966 Errors.push_back(EI.message());
967 });
968 return join(Errors.begin(), Errors.end(), "\n");
969}
970
971/// Consume a Error without doing anything. This method should be used
972/// only where an error can be considered a reasonable and expected return
973/// value.
974///
975/// Uses of this method are potentially indicative of design problems: If it's
976/// legitimate to do nothing while processing an "error", the error-producer
977/// might be more clearly refactored to return an Optional<T>.
978inline void consumeError(Error Err) {
979 handleAllErrors(std::move(Err), [](const ErrorInfoBase &) {});
980}
981
982/// Helper for converting an Error to a bool.
983///
984/// This method returns true if Err is in an error state, or false if it is
985/// in a success state. Puts Err in a checked state in both cases (unlike
986/// Error::operator bool(), which only does this for success states).
987inline bool errorToBool(Error Err) {
988 bool IsError = static_cast<bool>(Err);
989 if (IsError)
990 consumeError(std::move(Err));
991 return IsError;
992}
993
994/// Helper for Errors used as out-parameters.
995///
996/// This helper is for use with the Error-as-out-parameter idiom, where an error
997/// is passed to a function or method by reference, rather than being returned.
998/// In such cases it is helpful to set the checked bit on entry to the function
999/// so that the error can be written to (unchecked Errors abort on assignment)
1000/// and clear the checked bit on exit so that clients cannot accidentally forget
1001/// to check the result. This helper performs these actions automatically using
1002/// RAII:
1003///
1004/// @code{.cpp}
1005/// Result foo(Error &Err) {
1006/// ErrorAsOutParameter ErrAsOutParam(&Err); // 'Checked' flag set
1007/// // <body of foo>
1008/// // <- 'Checked' flag auto-cleared when ErrAsOutParam is destructed.
1009/// }
1010/// @endcode
1011///
1012/// ErrorAsOutParameter takes an Error* rather than Error& so that it can be
1013/// used with optional Errors (Error pointers that are allowed to be null). If
1014/// ErrorAsOutParameter took an Error reference, an instance would have to be
1015/// created inside every condition that verified that Error was non-null. By
1016/// taking an Error pointer we can just create one instance at the top of the
1017/// function.
1018class ErrorAsOutParameter {
1019public:
1020 ErrorAsOutParameter(Error *Err) : Err(Err) {
1021 // Raise the checked bit if Err is success.
1022 if (Err)
1023 (void)!!*Err;
1024 }
1025
1026 ~ErrorAsOutParameter() {
1027 // Clear the checked bit.
1028 if (Err && !*Err)
1029 *Err = Error::success();
1030 }
1031
1032private:
1033 Error *Err;
1034};
1035
1036/// Helper for Expected<T>s used as out-parameters.
1037///
1038/// See ErrorAsOutParameter.
1039template <typename T>
1040class ExpectedAsOutParameter {
1041public:
1042 ExpectedAsOutParameter(Expected<T> *ValOrErr)
1043 : ValOrErr(ValOrErr) {
1044 if (ValOrErr)
1045 (void)!!*ValOrErr;
1046 }
1047
1048 ~ExpectedAsOutParameter() {
1049 if (ValOrErr)
1050 ValOrErr->setUnchecked();
1051 }
1052
1053private:
1054 Expected<T> *ValOrErr;
1055};
1056
1057/// This class wraps a std::error_code in a Error.
1058///
1059/// This is useful if you're writing an interface that returns a Error
1060/// (or Expected) and you want to call code that still returns
1061/// std::error_codes.
1062class ECError : public ErrorInfo<ECError> {
1063 friend Error errorCodeToError(std::error_code);
1064
1065public:
1066 void setErrorCode(std::error_code EC) { this->EC = EC; }
1067 std::error_code convertToErrorCode() const override { return EC; }
1068 void log(raw_ostream &OS) const override { OS << EC.message(); }
1069
1070 // Used by ErrorInfo::classID.
1071 static char ID;
1072
1073protected:
1074 ECError() = default;
1075 ECError(std::error_code EC) : EC(EC) {}
1076
1077 std::error_code EC;
1078};
1079
1080/// The value returned by this function can be returned from convertToErrorCode
1081/// for Error values where no sensible translation to std::error_code exists.
1082/// It should only be used in this situation, and should never be used where a
1083/// sensible conversion to std::error_code is available, as attempts to convert
1084/// to/from this error will result in a fatal error. (i.e. it is a programmatic
1085///error to try to convert such a value).
1086std::error_code inconvertibleErrorCode();
1087
1088/// Helper for converting an std::error_code to a Error.
1089Error errorCodeToError(std::error_code EC);
1090
1091/// Helper for converting an ECError to a std::error_code.
1092///
1093/// This method requires that Err be Error() or an ECError, otherwise it
1094/// will trigger a call to abort().
1095std::error_code errorToErrorCode(Error Err);
1096
1097/// Convert an ErrorOr<T> to an Expected<T>.
1098template <typename T> Expected<T> errorOrToExpected(ErrorOr<T> &&EO) {
1099 if (auto EC = EO.getError())
1100 return errorCodeToError(EC);
1101 return std::move(*EO);
1102}
1103
1104/// Convert an Expected<T> to an ErrorOr<T>.
1105template <typename T> ErrorOr<T> expectedToErrorOr(Expected<T> &&E) {
1106 if (auto Err = E.takeError())
1107 return errorToErrorCode(std::move(Err));
1108 return std::move(*E);
1109}
1110
1111/// This class wraps a string in an Error.
1112///
1113/// StringError is useful in cases where the client is not expected to be able
1114/// to consume the specific error message programmatically (for example, if the
1115/// error message is to be presented to the user).
1116///
1117/// StringError can also be used when additional information is to be printed
1118/// along with a error_code message. Depending on the constructor called, this
1119/// class can either display:
1120/// 1. the error_code message (ECError behavior)
1121/// 2. a string
1122/// 3. the error_code message and a string
1123///
1124/// These behaviors are useful when subtyping is required; for example, when a
1125/// specific library needs an explicit error type. In the example below,
1126/// PDBError is derived from StringError:
1127///
1128/// @code{.cpp}
1129/// Expected<int> foo() {
1130/// return llvm::make_error<PDBError>(pdb_error_code::dia_failed_loading,
1131/// "Additional information");
1132/// }
1133/// @endcode
1134///
1135class StringError : public ErrorInfo<StringError> {
1136public:
1137 static char ID;
1138
1139 // Prints EC + S and converts to EC
1140 StringError(std::error_code EC, const Twine &S = Twine());
1141
1142 // Prints S and converts to EC
1143 StringError(const Twine &S, std::error_code EC);
1144
1145 void log(raw_ostream &OS) const override;
1146 std::error_code convertToErrorCode() const override;
1147
1148 const std::string &getMessage() const { return Msg; }
1149
1150private:
1151 std::string Msg;
1152 std::error_code EC;
1153 const bool PrintMsgOnly = false;
1154};
1155
1156/// Create formatted StringError object.
1157template <typename... Ts>
1158Error createStringError(std::error_code EC, char const *Fmt,
1159 const Ts &... Vals) {
1160 std::string Buffer;
1161 raw_string_ostream Stream(Buffer);
1162 Stream << format(Fmt, Vals...);
1163 return make_error<StringError>(Stream.str(), EC);
1164}
1165
1166Error createStringError(std::error_code EC, char const *Msg);
1167
1168/// This class wraps a filename and another Error.
1169///
1170/// In some cases, an error needs to live along a 'source' name, in order to
1171/// show more detailed information to the user.
1172class FileError final : public ErrorInfo<FileError> {
1173
1174 friend Error createFileError(std::string, Error);
1175
1176public:
1177 void log(raw_ostream &OS) const override {
1178 assert(Err && !FileName.empty() && "Trying to log after takeError().")((Err && !FileName.empty() && "Trying to log after takeError()."
) ? static_cast<void> (0) : __assert_fail ("Err && !FileName.empty() && \"Trying to log after takeError().\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 1178, __PRETTY_FUNCTION__))
;
1179 OS << "'" << FileName << "': ";
1180 Err->log(OS);
1181 }
1182
1183 Error takeError() { return Error(std::move(Err)); }
1184
1185 std::error_code convertToErrorCode() const override;
1186
1187 // Used by ErrorInfo::classID.
1188 static char ID;
1189
1190private:
1191 FileError(std::string F, std::unique_ptr<ErrorInfoBase> E) {
1192 assert(E && "Cannot create FileError from Error success value.")((E && "Cannot create FileError from Error success value."
) ? static_cast<void> (0) : __assert_fail ("E && \"Cannot create FileError from Error success value.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 1192, __PRETTY_FUNCTION__))
;
1193 assert(!F.empty() &&((!F.empty() && "The file name provided to FileError must not be empty."
) ? static_cast<void> (0) : __assert_fail ("!F.empty() && \"The file name provided to FileError must not be empty.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 1194, __PRETTY_FUNCTION__))
1194 "The file name provided to FileError must not be empty.")((!F.empty() && "The file name provided to FileError must not be empty."
) ? static_cast<void> (0) : __assert_fail ("!F.empty() && \"The file name provided to FileError must not be empty.\""
, "/build/llvm-toolchain-snapshot-8~svn345461/include/llvm/Support/Error.h"
, 1194, __PRETTY_FUNCTION__))
;
1195 FileName = F;
1196 Err = std::move(E);
1197 }
1198
1199 static Error build(std::string F, Error E) {
1200 return Error(std::unique_ptr<FileError>(new FileError(F, E.takePayload())));
1201 }
1202
1203 std::string FileName;
1204 std::unique_ptr<ErrorInfoBase> Err;
1205};
1206
1207/// Concatenate a source file path and/or name with an Error. The resulting
1208/// Error is unchecked.
1209inline Error createFileError(std::string F, Error E) {
1210 return FileError::build(F, std::move(E));
1211}
1212
1213Error createFileError(std::string F, ErrorSuccess) = delete;
1214
1215/// Helper for check-and-exit error handling.
1216///
1217/// For tool use only. NOT FOR USE IN LIBRARY CODE.
1218///
1219class ExitOnError {
1220public:
1221 /// Create an error on exit helper.
1222 ExitOnError(std::string Banner = "", int DefaultErrorExitCode = 1)
1223 : Banner(std::move(Banner)),
1224 GetExitCode([=](const Error &) { return DefaultErrorExitCode; }) {}
1225
1226 /// Set the banner string for any errors caught by operator().
1227 void setBanner(std::string Banner) { this->Banner = std::move(Banner); }
1228
1229 /// Set the exit-code mapper function.
1230 void setExitCodeMapper(std::function<int(const Error &)> GetExitCode) {
1231 this->GetExitCode = std::move(GetExitCode);
1232 }
1233
1234 /// Check Err. If it's in a failure state log the error(s) and exit.
1235 void operator()(Error Err) const { checkError(std::move(Err)); }
1236
1237 /// Check E. If it's in a success state then return the contained value. If
1238 /// it's in a failure state log the error(s) and exit.
1239 template <typename T> T operator()(Expected<T> &&E) const {
1240 checkError(E.takeError());
1241 return std::move(*E);
1242 }
1243
1244 /// Check E. If it's in a success state then return the contained reference. If
1245 /// it's in a failure state log the error(s) and exit.
1246 template <typename T> T& operator()(Expected<T&> &&E) const {
1247 checkError(E.takeError());
1248 return *E;
1249 }
1250
1251private:
1252 void checkError(Error Err) const {
1253 if (Err) {
1254 int ExitCode = GetExitCode(Err);
1255 logAllUnhandledErrors(std::move(Err), errs(), Banner);
1256 exit(ExitCode);
1257 }
1258 }
1259
1260 std::string Banner;
1261 std::function<int(const Error &)> GetExitCode;
1262};
1263
1264/// Conversion from Error to LLVMErrorRef for C error bindings.
1265inline LLVMErrorRef wrap(Error Err) {
1266 return reinterpret_cast<LLVMErrorRef>(Err.takePayload().release());
1267}
1268
1269/// Conversion from LLVMErrorRef to Error for C error bindings.
1270inline Error unwrap(LLVMErrorRef ErrRef) {
1271 return Error(std::unique_ptr<ErrorInfoBase>(
1272 reinterpret_cast<ErrorInfoBase *>(ErrRef)));
1273}
1274
1275} // end namespace llvm
1276
1277#endif // LLVM_SUPPORT_ERROR_H