Bug Summary

File:build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
Warning:line 191, column 9
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name InstrProfiling.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Transforms/Instrumentation -I /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/lib/Transforms/Instrumentation -I include -I /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm=build-llvm -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm=build-llvm -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -O3 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/build-llvm=build-llvm -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-09-04-125545-48738-1 -x c++ /build/llvm-toolchain-snapshot-16~++20220904122748+c444af1c20b3/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/Triple.h"
20#include "llvm/ADT/Twine.h"
21#include "llvm/Analysis/BlockFrequencyInfo.h"
22#include "llvm/Analysis/BranchProbabilityInfo.h"
23#include "llvm/Analysis/LoopInfo.h"
24#include "llvm/Analysis/TargetLibraryInfo.h"
25#include "llvm/IR/Attributes.h"
26#include "llvm/IR/BasicBlock.h"
27#include "llvm/IR/Constant.h"
28#include "llvm/IR/Constants.h"
29#include "llvm/IR/DIBuilder.h"
30#include "llvm/IR/DerivedTypes.h"
31#include "llvm/IR/DiagnosticInfo.h"
32#include "llvm/IR/Dominators.h"
33#include "llvm/IR/Function.h"
34#include "llvm/IR/GlobalValue.h"
35#include "llvm/IR/GlobalVariable.h"
36#include "llvm/IR/IRBuilder.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/IntrinsicInst.h"
40#include "llvm/IR/Module.h"
41#include "llvm/IR/Type.h"
42#include "llvm/InitializePasses.h"
43#include "llvm/Pass.h"
44#include "llvm/ProfileData/InstrProf.h"
45#include "llvm/ProfileData/InstrProfCorrelator.h"
46#include "llvm/Support/Casting.h"
47#include "llvm/Support/CommandLine.h"
48#include "llvm/Support/Error.h"
49#include "llvm/Support/ErrorHandling.h"
50#include "llvm/Transforms/Utils/ModuleUtils.h"
51#include "llvm/Transforms/Utils/SSAUpdater.h"
52#include <algorithm>
53#include <cassert>
54#include <cstdint>
55#include <string>
56
57using namespace llvm;
58
59#define DEBUG_TYPE"instrprof" "instrprof"
60
61namespace llvm {
62cl::opt<bool>
63 DebugInfoCorrelate("debug-info-correlate",
64 cl::desc("Use debug info to correlate profiles."),
65 cl::init(false));
66} // namespace llvm
67
68namespace {
69
70cl::opt<bool> DoHashBasedCounterSplit(
71 "hash-based-counter-split",
72 cl::desc("Rename counter variable of a comdat function based on cfg hash"),
73 cl::init(true));
74
75cl::opt<bool>
76 RuntimeCounterRelocation("runtime-counter-relocation",
77 cl::desc("Enable relocating counters at runtime."),
78 cl::init(false));
79
80cl::opt<bool> ValueProfileStaticAlloc(
81 "vp-static-alloc",
82 cl::desc("Do static counter allocation for value profiler"),
83 cl::init(true));
84
85cl::opt<double> NumCountersPerValueSite(
86 "vp-counters-per-site",
87 cl::desc("The average number of profile counters allocated "
88 "per value profiling site."),
89 // This is set to a very small value because in real programs, only
90 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
91 // For those sites with non-zero profile, the average number of targets
92 // is usually smaller than 2.
93 cl::init(1.0));
94
95cl::opt<bool> AtomicCounterUpdateAll(
96 "instrprof-atomic-counter-update-all",
97 cl::desc("Make all profile counter updates atomic (for testing only)"),
98 cl::init(false));
99
100cl::opt<bool> AtomicCounterUpdatePromoted(
101 "atomic-counter-update-promoted",
102 cl::desc("Do counter update using atomic fetch add "
103 " for promoted counters only"),
104 cl::init(false));
105
106cl::opt<bool> AtomicFirstCounter(
107 "atomic-first-counter",
108 cl::desc("Use atomic fetch add for first counter in a function (usually "
109 "the entry counter)"),
110 cl::init(false));
111
112// If the option is not specified, the default behavior about whether
113// counter promotion is done depends on how instrumentaiton lowering
114// pipeline is setup, i.e., the default value of true of this option
115// does not mean the promotion will be done by default. Explicitly
116// setting this option can override the default behavior.
117cl::opt<bool> DoCounterPromotion("do-counter-promotion",
118 cl::desc("Do counter register promotion"),
119 cl::init(false));
120cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
121 "max-counter-promotions-per-loop", cl::init(20),
122 cl::desc("Max number counter promotions per loop to avoid"
123 " increasing register pressure too much"));
124
125// A debug option
126cl::opt<int>
127 MaxNumOfPromotions("max-counter-promotions", cl::init(-1),
128 cl::desc("Max number of allowed counter promotions"));
129
130cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
131 "speculative-counter-promotion-max-exiting", cl::init(3),
132 cl::desc("The max number of exiting blocks of a loop to allow "
133 " speculative counter promotion"));
134
135cl::opt<bool> SpeculativeCounterPromotionToLoop(
136 "speculative-counter-promotion-to-loop",
137 cl::desc("When the option is false, if the target block is in a loop, "
138 "the promotion will be disallowed unless the promoted counter "
139 " update can be further/iteratively promoted into an acyclic "
140 " region."));
141
142cl::opt<bool> IterativeCounterPromotion(
143 "iterative-counter-promotion", cl::init(true),
144 cl::desc("Allow counter promotion across the whole loop nest."));
145
146cl::opt<bool> SkipRetExitBlock(
147 "skip-ret-exit-block", cl::init(true),
148 cl::desc("Suppress counter promotion if exit blocks contain ret."));
149
150///
151/// A helper class to promote one counter RMW operation in the loop
152/// into register update.
153///
154/// RWM update for the counter will be sinked out of the loop after
155/// the transformation.
156///
157class PGOCounterPromoterHelper : public LoadAndStorePromoter {
158public:
159 PGOCounterPromoterHelper(
160 Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,
161 BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
162 ArrayRef<Instruction *> InsertPts,
163 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
164 LoopInfo &LI)
165 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
166 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
167 assert(isa<LoadInst>(L))(static_cast <bool> (isa<LoadInst>(L)) ? void (0)
: __assert_fail ("isa<LoadInst>(L)", "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp"
, 167, __extension__ __PRETTY_FUNCTION__))
;
168 assert(isa<StoreInst>(S))(static_cast <bool> (isa<StoreInst>(S)) ? void (0
) : __assert_fail ("isa<StoreInst>(S)", "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp"
, 168, __extension__ __PRETTY_FUNCTION__))
;
169 SSA.AddAvailableValue(PH, Init);
170 }
171
172 void doExtraRewritesBeforeFinalDeletion() override {
173 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
1
Assuming 'i' is not equal to 'e'
2
Loop condition is true. Entering loop body
174 BasicBlock *ExitBlock = ExitBlocks[i];
175 Instruction *InsertPos = InsertPts[i];
176 // Get LiveIn value into the ExitBlock. If there are multiple
177 // predecessors, the value is defined by a PHI node in this
178 // block.
179 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
180 Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
3
Field 'Store' is a 'CastReturnType'
181 Type *Ty = LiveInValue->getType();
182 IRBuilder<> Builder(InsertPos);
183 if (auto *AddrInst
4.1
'AddrInst' is non-null
= dyn_cast_or_null<IntToPtrInst>(Addr)) {
4
Assuming 'Addr' is a 'CastReturnType'
5
Taking true branch
184 // If isRuntimeCounterRelocationEnabled() is true then the address of
185 // the store instruction is computed with two instructions in
186 // InstrProfiling::getCounterAddress(). We need to copy those
187 // instructions to this block to compute Addr correctly.
188 // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
189 // %Addr = inttoptr i64 %BiasAdd to i64*
190 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
6
Assuming the object is not a 'CastReturnType'
7
'OrigBiasInst' initialized to a null pointer value
191 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add)(static_cast <bool> (OrigBiasInst->getOpcode() == Instruction
::BinaryOps::Add) ? void (0) : __assert_fail ("OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add"
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 191
, __extension__ __PRETTY_FUNCTION__))
;
8
Called C++ object pointer is null
192 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
193 Addr = Builder.CreateIntToPtr(BiasInst, Ty->getPointerTo());
194 }
195 if (AtomicCounterUpdatePromoted)
196 // automic update currently can only be promoted across the current
197 // loop, not the whole loop nest.
198 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
199 MaybeAlign(),
200 AtomicOrdering::SequentiallyConsistent);
201 else {
202 LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
203 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
204 auto *NewStore = Builder.CreateStore(NewVal, Addr);
205
206 // Now update the parent loop's candidate list:
207 if (IterativeCounterPromotion) {
208 auto *TargetLoop = LI.getLoopFor(ExitBlock);
209 if (TargetLoop)
210 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
211 }
212 }
213 }
214 }
215
216private:
217 Instruction *Store;
218 ArrayRef<BasicBlock *> ExitBlocks;
219 ArrayRef<Instruction *> InsertPts;
220 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
221 LoopInfo &LI;
222};
223
224/// A helper class to do register promotion for all profile counter
225/// updates in a loop.
226///
227class PGOCounterPromoter {
228public:
229 PGOCounterPromoter(
230 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
231 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
232 : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {
233
234 // Skip collection of ExitBlocks and InsertPts for loops that will not be
235 // able to have counters promoted.
236 SmallVector<BasicBlock *, 8> LoopExitBlocks;
237 SmallPtrSet<BasicBlock *, 8> BlockSet;
238
239 L.getExitBlocks(LoopExitBlocks);
240 if (!isPromotionPossible(&L, LoopExitBlocks))
241 return;
242
243 for (BasicBlock *ExitBlock : LoopExitBlocks) {
244 if (BlockSet.insert(ExitBlock).second) {
245 ExitBlocks.push_back(ExitBlock);
246 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
247 }
248 }
249 }
250
251 bool run(int64_t *NumPromoted) {
252 // Skip 'infinite' loops:
253 if (ExitBlocks.size() == 0)
254 return false;
255
256 // Skip if any of the ExitBlocks contains a ret instruction.
257 // This is to prevent dumping of incomplete profile -- if the
258 // the loop is a long running loop and dump is called in the middle
259 // of the loop, the result profile is incomplete.
260 // FIXME: add other heuristics to detect long running loops.
261 if (SkipRetExitBlock) {
262 for (auto *BB : ExitBlocks)
263 if (isa<ReturnInst>(BB->getTerminator()))
264 return false;
265 }
266
267 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
268 if (MaxProm == 0)
269 return false;
270
271 unsigned Promoted = 0;
272 for (auto &Cand : LoopToCandidates[&L]) {
273
274 SmallVector<PHINode *, 4> NewPHIs;
275 SSAUpdater SSA(&NewPHIs);
276 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
277
278 // If BFI is set, we will use it to guide the promotions.
279 if (BFI) {
280 auto *BB = Cand.first->getParent();
281 auto InstrCount = BFI->getBlockProfileCount(BB);
282 if (!InstrCount)
283 continue;
284 auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
285 // If the average loop trip count is not greater than 1.5, we skip
286 // promotion.
287 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
288 continue;
289 }
290
291 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
292 L.getLoopPreheader(), ExitBlocks,
293 InsertPts, LoopToCandidates, LI);
294 Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
295 Promoted++;
296 if (Promoted >= MaxProm)
297 break;
298
299 (*NumPromoted)++;
300 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
301 break;
302 }
303
304 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("instrprof")) { dbgs() << Promoted << " counters promoted for loop (depth="
<< L.getLoopDepth() << ")\n"; } } while (false)
305 << L.getLoopDepth() << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("instrprof")) { dbgs() << Promoted << " counters promoted for loop (depth="
<< L.getLoopDepth() << ")\n"; } } while (false)
;
306 return Promoted != 0;
307 }
308
309private:
310 bool allowSpeculativeCounterPromotion(Loop *LP) {
311 SmallVector<BasicBlock *, 8> ExitingBlocks;
312 L.getExitingBlocks(ExitingBlocks);
313 // Not considierered speculative.
314 if (ExitingBlocks.size() == 1)
315 return true;
316 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
317 return false;
318 return true;
319 }
320
321 // Check whether the loop satisfies the basic conditions needed to perform
322 // Counter Promotions.
323 bool
324 isPromotionPossible(Loop *LP,
325 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
326 // We can't insert into a catchswitch.
327 if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
328 return isa<CatchSwitchInst>(Exit->getTerminator());
329 }))
330 return false;
331
332 if (!LP->hasDedicatedExits())
333 return false;
334
335 BasicBlock *PH = LP->getLoopPreheader();
336 if (!PH)
337 return false;
338
339 return true;
340 }
341
342 // Returns the max number of Counter Promotions for LP.
343 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
344 SmallVector<BasicBlock *, 8> LoopExitBlocks;
345 LP->getExitBlocks(LoopExitBlocks);
346 if (!isPromotionPossible(LP, LoopExitBlocks))
347 return 0;
348
349 SmallVector<BasicBlock *, 8> ExitingBlocks;
350 LP->getExitingBlocks(ExitingBlocks);
351
352 // If BFI is set, we do more aggressive promotions based on BFI.
353 if (BFI)
354 return (unsigned)-1;
355
356 // Not considierered speculative.
357 if (ExitingBlocks.size() == 1)
358 return MaxNumOfPromotionsPerLoop;
359
360 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
361 return 0;
362
363 // Whether the target block is in a loop does not matter:
364 if (SpeculativeCounterPromotionToLoop)
365 return MaxNumOfPromotionsPerLoop;
366
367 // Now check the target block:
368 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
369 for (auto *TargetBlock : LoopExitBlocks) {
370 auto *TargetLoop = LI.getLoopFor(TargetBlock);
371 if (!TargetLoop)
372 continue;
373 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
374 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
375 MaxProm =
376 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
377 PendingCandsInTarget);
378 }
379 return MaxProm;
380 }
381
382 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
383 SmallVector<BasicBlock *, 8> ExitBlocks;
384 SmallVector<Instruction *, 8> InsertPts;
385 Loop &L;
386 LoopInfo &LI;
387 BlockFrequencyInfo *BFI;
388};
389
390enum class ValueProfilingCallType {
391 // Individual values are tracked. Currently used for indiret call target
392 // profiling.
393 Default,
394
395 // MemOp: the memop size value profiling.
396 MemOp
397};
398
399} // end anonymous namespace
400
401PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
402 FunctionAnalysisManager &FAM =
403 AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
404 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
405 return FAM.getResult<TargetLibraryAnalysis>(F);
406 };
407 if (!run(M, GetTLI))
408 return PreservedAnalyses::all();
409
410 return PreservedAnalyses::none();
411}
412
413bool InstrProfiling::lowerIntrinsics(Function *F) {
414 bool MadeChange = false;
415 PromotionCandidates.clear();
416 for (BasicBlock &BB : *F) {
417 for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
418 if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
419 lowerIncrement(IPIS);
420 MadeChange = true;
421 } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
422 lowerIncrement(IPI);
423 MadeChange = true;
424 } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
425 lowerCover(IPC);
426 MadeChange = true;
427 } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
428 lowerValueProfileInst(IPVP);
429 MadeChange = true;
430 }
431 }
432 }
433
434 if (!MadeChange)
435 return false;
436
437 promoteCounterLoadStores(F);
438 return true;
439}
440
441bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
442 // Mach-O don't support weak external references.
443 if (TT.isOSBinFormatMachO())
444 return false;
445
446 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
447 return RuntimeCounterRelocation;
448
449 // Fuchsia uses runtime counter relocation by default.
450 return TT.isOSFuchsia();
451}
452
453bool InstrProfiling::isCounterPromotionEnabled() const {
454 if (DoCounterPromotion.getNumOccurrences() > 0)
455 return DoCounterPromotion;
456
457 return Options.DoCounterPromotion;
458}
459
460void InstrProfiling::promoteCounterLoadStores(Function *F) {
461 if (!isCounterPromotionEnabled())
462 return;
463
464 DominatorTree DT(*F);
465 LoopInfo LI(DT);
466 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
467
468 std::unique_ptr<BlockFrequencyInfo> BFI;
469 if (Options.UseBFIInPromotion) {
470 std::unique_ptr<BranchProbabilityInfo> BPI;
471 BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
472 BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
473 }
474
475 for (const auto &LoadStore : PromotionCandidates) {
476 auto *CounterLoad = LoadStore.first;
477 auto *CounterStore = LoadStore.second;
478 BasicBlock *BB = CounterLoad->getParent();
479 Loop *ParentLoop = LI.getLoopFor(BB);
480 if (!ParentLoop)
481 continue;
482 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
483 }
484
485 SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
486
487 // Do a post-order traversal of the loops so that counter updates can be
488 // iteratively hoisted outside the loop nest.
489 for (auto *Loop : llvm::reverse(Loops)) {
490 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
491 Promoter.run(&TotalCountersPromoted);
492 }
493}
494
495static bool needsRuntimeHookUnconditionally(const Triple &TT) {
496 // On Fuchsia, we only need runtime hook if any counters are present.
497 if (TT.isOSFuchsia())
498 return false;
499
500 return true;
501}
502
503/// Check if the module contains uses of any profiling intrinsics.
504static bool containsProfilingIntrinsics(Module &M) {
505 auto containsIntrinsic = [&](int ID) {
506 if (auto *F = M.getFunction(Intrinsic::getName(ID)))
507 return !F->use_empty();
508 return false;
509 };
510 return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
511 containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
512 containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
513 containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
514}
515
516bool InstrProfiling::run(
517 Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
518 this->M = &M;
519 this->GetTLI = std::move(GetTLI);
520 NamesVar = nullptr;
521 NamesSize = 0;
522 ProfileDataMap.clear();
523 CompilerUsedVars.clear();
524 UsedVars.clear();
525 TT = Triple(M.getTargetTriple());
526
527 bool MadeChange = false;
528
529 // Emit the runtime hook even if no counters are present.
530 if (needsRuntimeHookUnconditionally(TT))
531 MadeChange = emitRuntimeHook();
532
533 // Improve compile time by avoiding linear scans when there is no work.
534 GlobalVariable *CoverageNamesVar =
535 M.getNamedGlobal(getCoverageUnusedNamesVarName());
536 if (!containsProfilingIntrinsics(M) && !CoverageNamesVar)
537 return MadeChange;
538
539 // We did not know how many value sites there would be inside
540 // the instrumented function. This is counting the number of instrumented
541 // target value sites to enter it as field in the profile data variable.
542 for (Function &F : M) {
543 InstrProfIncrementInst *FirstProfIncInst = nullptr;
544 for (BasicBlock &BB : F)
545 for (auto I = BB.begin(), E = BB.end(); I != E; I++)
546 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
547 computeNumValueSiteCounts(Ind);
548 else if (FirstProfIncInst == nullptr)
549 FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
550
551 // Value profiling intrinsic lowering requires per-function profile data
552 // variable to be created first.
553 if (FirstProfIncInst != nullptr)
554 static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
555 }
556
557 for (Function &F : M)
558 MadeChange |= lowerIntrinsics(&F);
559
560 if (CoverageNamesVar) {
561 lowerCoverageData(CoverageNamesVar);
562 MadeChange = true;
563 }
564
565 if (!MadeChange)
566 return false;
567
568 emitVNodes();
569 emitNameData();
570 emitRuntimeHook();
571 emitRegistration();
572 emitUses();
573 emitInitialization();
574 return true;
575}
576
577static FunctionCallee getOrInsertValueProfilingCall(
578 Module &M, const TargetLibraryInfo &TLI,
579 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
580 LLVMContext &Ctx = M.getContext();
581 auto *ReturnTy = Type::getVoidTy(M.getContext());
582
583 AttributeList AL;
584 if (auto AK = TLI.getExtAttrForI32Param(false))
585 AL = AL.addParamAttribute(M.getContext(), 2, AK);
586
587 assert((CallType == ValueProfilingCallType::Default ||(static_cast <bool> ((CallType == ValueProfilingCallType
::Default || CallType == ValueProfilingCallType::MemOp) &&
"Must be Default or MemOp") ? void (0) : __assert_fail ("(CallType == ValueProfilingCallType::Default || CallType == ValueProfilingCallType::MemOp) && \"Must be Default or MemOp\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 589
, __extension__ __PRETTY_FUNCTION__))
588 CallType == ValueProfilingCallType::MemOp) &&(static_cast <bool> ((CallType == ValueProfilingCallType
::Default || CallType == ValueProfilingCallType::MemOp) &&
"Must be Default or MemOp") ? void (0) : __assert_fail ("(CallType == ValueProfilingCallType::Default || CallType == ValueProfilingCallType::MemOp) && \"Must be Default or MemOp\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 589
, __extension__ __PRETTY_FUNCTION__))
589 "Must be Default or MemOp")(static_cast <bool> ((CallType == ValueProfilingCallType
::Default || CallType == ValueProfilingCallType::MemOp) &&
"Must be Default or MemOp") ? void (0) : __assert_fail ("(CallType == ValueProfilingCallType::Default || CallType == ValueProfilingCallType::MemOp) && \"Must be Default or MemOp\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 589
, __extension__ __PRETTY_FUNCTION__))
;
590 Type *ParamTypes[] = {
591#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
592#include "llvm/ProfileData/InstrProfData.inc"
593 };
594 auto *ValueProfilingCallTy =
595 FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
596 StringRef FuncName = CallType == ValueProfilingCallType::Default
597 ? getInstrProfValueProfFuncName()
598 : getInstrProfValueProfMemOpFuncName();
599 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
600}
601
602void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
603 GlobalVariable *Name = Ind->getName();
604 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
605 uint64_t Index = Ind->getIndex()->getZExtValue();
606 auto &PD = ProfileDataMap[Name];
607 PD.NumValueSites[ValueKind] =
608 std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));
609}
610
611void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
612 // TODO: Value profiling heavily depends on the data section which is omitted
613 // in lightweight mode. We need to move the value profile pointer to the
614 // Counter struct to get this working.
615 assert((static_cast <bool> (!DebugInfoCorrelate && "Value profiling is not yet supported with lightweight instrumentation"
) ? void (0) : __assert_fail ("!DebugInfoCorrelate && \"Value profiling is not yet supported with lightweight instrumentation\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 617
, __extension__ __PRETTY_FUNCTION__))
616 !DebugInfoCorrelate &&(static_cast <bool> (!DebugInfoCorrelate && "Value profiling is not yet supported with lightweight instrumentation"
) ? void (0) : __assert_fail ("!DebugInfoCorrelate && \"Value profiling is not yet supported with lightweight instrumentation\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 617
, __extension__ __PRETTY_FUNCTION__))
617 "Value profiling is not yet supported with lightweight instrumentation")(static_cast <bool> (!DebugInfoCorrelate && "Value profiling is not yet supported with lightweight instrumentation"
) ? void (0) : __assert_fail ("!DebugInfoCorrelate && \"Value profiling is not yet supported with lightweight instrumentation\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 617
, __extension__ __PRETTY_FUNCTION__))
;
618 GlobalVariable *Name = Ind->getName();
619 auto It = ProfileDataMap.find(Name);
620 assert(It != ProfileDataMap.end() && It->second.DataVar &&(static_cast <bool> (It != ProfileDataMap.end() &&
It->second.DataVar && "value profiling detected in function with no counter incerement"
) ? void (0) : __assert_fail ("It != ProfileDataMap.end() && It->second.DataVar && \"value profiling detected in function with no counter incerement\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 621
, __extension__ __PRETTY_FUNCTION__))
621 "value profiling detected in function with no counter incerement")(static_cast <bool> (It != ProfileDataMap.end() &&
It->second.DataVar && "value profiling detected in function with no counter incerement"
) ? void (0) : __assert_fail ("It != ProfileDataMap.end() && It->second.DataVar && \"value profiling detected in function with no counter incerement\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 621
, __extension__ __PRETTY_FUNCTION__))
;
622
623 GlobalVariable *DataVar = It->second.DataVar;
624 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
625 uint64_t Index = Ind->getIndex()->getZExtValue();
626 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
627 Index += It->second.NumValueSites[Kind];
628
629 IRBuilder<> Builder(Ind);
630 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
631 llvm::InstrProfValueKind::IPVK_MemOPSize);
632 CallInst *Call = nullptr;
633 auto *TLI = &GetTLI(*Ind->getFunction());
634
635 // To support value profiling calls within Windows exception handlers, funclet
636 // information contained within operand bundles needs to be copied over to
637 // the library call. This is required for the IR to be processed by the
638 // WinEHPrepare pass.
639 SmallVector<OperandBundleDef, 1> OpBundles;
640 Ind->getOperandBundlesAsDefs(OpBundles);
641 if (!IsMemOpSize) {
642 Value *Args[3] = {Ind->getTargetValue(),
643 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
644 Builder.getInt32(Index)};
645 Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
646 OpBundles);
647 } else {
648 Value *Args[3] = {Ind->getTargetValue(),
649 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
650 Builder.getInt32(Index)};
651 Call = Builder.CreateCall(
652 getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
653 Args, OpBundles);
654 }
655 if (auto AK = TLI->getExtAttrForI32Param(false))
656 Call->addParamAttr(2, AK);
657 Ind->replaceAllUsesWith(Call);
658 Ind->eraseFromParent();
659}
660
661Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) {
662 auto *Counters = getOrCreateRegionCounters(I);
663 IRBuilder<> Builder(I);
664
665 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
666 Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());
667
668 if (!isRuntimeCounterRelocationEnabled())
669 return Addr;
670
671 Type *Int64Ty = Type::getInt64Ty(M->getContext());
672 Function *Fn = I->getParent()->getParent();
673 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
674 if (!BiasLI) {
675 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
676 auto *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
677 if (!Bias) {
678 // Compiler must define this variable when runtime counter relocation
679 // is being used. Runtime has a weak external reference that is used
680 // to check whether that's the case or not.
681 Bias = new GlobalVariable(
682 *M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
683 Constant::getNullValue(Int64Ty), getInstrProfCounterBiasVarName());
684 Bias->setVisibility(GlobalVariable::HiddenVisibility);
685 // A definition that's weak (linkonce_odr) without being in a COMDAT
686 // section wouldn't lead to link errors, but it would lead to a dead
687 // data word from every TU but one. Putting it in COMDAT ensures there
688 // will be exactly one data slot in the link.
689 if (TT.supportsCOMDAT())
690 Bias->setComdat(M->getOrInsertComdat(Bias->getName()));
691 }
692 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
693 }
694 auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
695 return Builder.CreateIntToPtr(Add, Addr->getType());
696}
697
698void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) {
699 auto *Addr = getCounterAddress(CoverInstruction);
700 IRBuilder<> Builder(CoverInstruction);
701 // We store zero to represent that this block is covered.
702 Builder.CreateStore(Builder.getInt8(0), Addr);
703 CoverInstruction->eraseFromParent();
704}
705
706void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
707 auto *Addr = getCounterAddress(Inc);
708
709 IRBuilder<> Builder(Inc);
710 if (Options.Atomic || AtomicCounterUpdateAll ||
711 (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {
712 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
713 MaybeAlign(), AtomicOrdering::Monotonic);
714 } else {
715 Value *IncStep = Inc->getStep();
716 Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
717 auto *Count = Builder.CreateAdd(Load, Inc->getStep());
718 auto *Store = Builder.CreateStore(Count, Addr);
719 if (isCounterPromotionEnabled())
720 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
721 }
722 Inc->eraseFromParent();
723}
724
725void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
726 ConstantArray *Names =
727 cast<ConstantArray>(CoverageNamesVar->getInitializer());
728 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
729 Constant *NC = Names->getOperand(I);
730 Value *V = NC->stripPointerCasts();
731 assert(isa<GlobalVariable>(V) && "Missing reference to function name")(static_cast <bool> (isa<GlobalVariable>(V) &&
"Missing reference to function name") ? void (0) : __assert_fail
("isa<GlobalVariable>(V) && \"Missing reference to function name\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 731
, __extension__ __PRETTY_FUNCTION__))
;
732 GlobalVariable *Name = cast<GlobalVariable>(V);
733
734 Name->setLinkage(GlobalValue::PrivateLinkage);
735 ReferencedNames.push_back(Name);
736 if (isa<ConstantExpr>(NC))
737 NC->dropAllReferences();
738 }
739 CoverageNamesVar->eraseFromParent();
740}
741
742/// Get the name of a profiling variable for a particular function.
743static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
744 bool &Renamed) {
745 StringRef NamePrefix = getInstrProfNameVarPrefix();
746 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
747 Function *F = Inc->getParent()->getParent();
748 Module *M = F->getParent();
749 if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
750 !canRenameComdatFunc(*F)) {
751 Renamed = false;
752 return (Prefix + Name).str();
753 }
754 Renamed = true;
755 uint64_t FuncHash = Inc->getHash()->getZExtValue();
756 SmallVector<char, 24> HashPostfix;
757 if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
758 return (Prefix + Name).str();
759 return (Prefix + Name + "." + Twine(FuncHash)).str();
760}
761
762static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {
763 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
764 if (!MD)
765 return 0;
766
767 // If the flag is a ConstantAsMetadata, it should be an integer representable
768 // in 64-bits.
769 return cast<ConstantInt>(MD->getValue())->getZExtValue();
770}
771
772static bool enablesValueProfiling(const Module &M) {
773 return isIRPGOFlagSet(&M) ||
774 getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;
775}
776
777// Conservatively returns true if data variables may be referenced by code.
778static bool profDataReferencedByCode(const Module &M) {
779 return enablesValueProfiling(M);
780}
781
782static inline bool shouldRecordFunctionAddr(Function *F) {
783 // Only record function addresses if IR PGO is enabled or if clang value
784 // profiling is enabled. Recording function addresses greatly increases object
785 // file size, because it prevents the inliner from deleting functions that
786 // have been inlined everywhere.
787 if (!profDataReferencedByCode(*F->getParent()))
788 return false;
789
790 // Check the linkage
791 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
792 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
793 !HasAvailableExternallyLinkage)
794 return true;
795
796 // A function marked 'alwaysinline' with available_externally linkage can't
797 // have its address taken. Doing so would create an undefined external ref to
798 // the function, which would fail to link.
799 if (HasAvailableExternallyLinkage &&
800 F->hasFnAttribute(Attribute::AlwaysInline))
801 return false;
802
803 // Prohibit function address recording if the function is both internal and
804 // COMDAT. This avoids the profile data variable referencing internal symbols
805 // in COMDAT.
806 if (F->hasLocalLinkage() && F->hasComdat())
807 return false;
808
809 // Check uses of this function for other than direct calls or invokes to it.
810 // Inline virtual functions have linkeOnceODR linkage. When a key method
811 // exists, the vtable will only be emitted in the TU where the key method
812 // is defined. In a TU where vtable is not available, the function won't
813 // be 'addresstaken'. If its address is not recorded here, the profile data
814 // with missing address may be picked by the linker leading to missing
815 // indirect call target info.
816 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
817}
818
819static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
820 // Don't do this for Darwin. compiler-rt uses linker magic.
821 if (TT.isOSDarwin())
822 return false;
823 // Use linker script magic to get data/cnts/name start/end.
824 if (TT.isOSAIX() || TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
825 TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS() || TT.isOSWindows())
826 return false;
827
828 return true;
829}
830
831GlobalVariable *
832InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name,
833 GlobalValue::LinkageTypes Linkage) {
834 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
835 auto &Ctx = M->getContext();
836 GlobalVariable *GV;
837 if (isa<InstrProfCoverInst>(Inc)) {
838 auto *CounterTy = Type::getInt8Ty(Ctx);
839 auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);
840 // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
841 std::vector<Constant *> InitialValues(NumCounters,
842 Constant::getAllOnesValue(CounterTy));
843 GV = new GlobalVariable(*M, CounterArrTy, false, Linkage,
844 ConstantArray::get(CounterArrTy, InitialValues),
845 Name);
846 GV->setAlignment(Align(1));
847 } else {
848 auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
849 GV = new GlobalVariable(*M, CounterTy, false, Linkage,
850 Constant::getNullValue(CounterTy), Name);
851 GV->setAlignment(Align(8));
852 }
853 return GV;
854}
855
856GlobalVariable *
857InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
858 GlobalVariable *NamePtr = Inc->getName();
859 auto &PD = ProfileDataMap[NamePtr];
860 if (PD.RegionCounters)
861 return PD.RegionCounters;
862
863 // Match the linkage and visibility of the name global.
864 Function *Fn = Inc->getParent()->getParent();
865 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
866 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
867
868 // Use internal rather than private linkage so the counter variable shows up
869 // in the symbol table when using debug info for correlation.
870 if (DebugInfoCorrelate && TT.isOSBinFormatMachO() &&
871 Linkage == GlobalValue::PrivateLinkage)
872 Linkage = GlobalValue::InternalLinkage;
873
874 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
875 // symbols in the same csect won't be discarded. When there are duplicate weak
876 // symbols, we can NOT guarantee that the relocations get resolved to the
877 // intended weak symbol, so we can not ensure the correctness of the relative
878 // CounterPtr, so we have to use private linkage for counter and data symbols.
879 if (TT.isOSBinFormatXCOFF()) {
880 Linkage = GlobalValue::PrivateLinkage;
881 Visibility = GlobalValue::DefaultVisibility;
882 }
883 // Move the name variable to the right section. Place them in a COMDAT group
884 // if the associated function is a COMDAT. This will make sure that only one
885 // copy of counters of the COMDAT function will be emitted after linking. Keep
886 // in mind that this pass may run before the inliner, so we need to create a
887 // new comdat group for the counters and profiling data. If we use the comdat
888 // of the parent function, that will result in relocations against discarded
889 // sections.
890 //
891 // If the data variable is referenced by code, counters and data have to be
892 // in different comdats for COFF because the Visual C++ linker will report
893 // duplicate symbol errors if there are multiple external symbols with the
894 // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
895 //
896 // For ELF, when not using COMDAT, put counters, data and values into a
897 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
898 // allows -z start-stop-gc to discard the entire group when the function is
899 // discarded.
900 bool DataReferencedByCode = profDataReferencedByCode(*M);
901 bool NeedComdat = needsComdatForCounter(*Fn, *M);
902 bool Renamed;
903 std::string CntsVarName =
904 getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed);
905 std::string DataVarName =
906 getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
907 auto MaybeSetComdat = [&](GlobalVariable *GV) {
908 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
909 if (UseComdat) {
910 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
911 ? GV->getName()
912 : CntsVarName;
913 Comdat *C = M->getOrInsertComdat(GroupName);
914 if (!NeedComdat)
915 C->setSelectionKind(Comdat::NoDeduplicate);
916 GV->setComdat(C);
917 // COFF doesn't allow the comdat group leader to have private linkage, so
918 // upgrade private linkage to internal linkage to produce a symbol table
919 // entry.
920 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
921 GV->setLinkage(GlobalValue::InternalLinkage);
922 }
923 };
924
925 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
926 LLVMContext &Ctx = M->getContext();
927
928 auto *CounterPtr = createRegionCounters(Inc, CntsVarName, Linkage);
929 CounterPtr->setVisibility(Visibility);
930 CounterPtr->setSection(
931 getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
932 CounterPtr->setLinkage(Linkage);
933 MaybeSetComdat(CounterPtr);
934 PD.RegionCounters = CounterPtr;
935 if (DebugInfoCorrelate) {
936 if (auto *SP = Fn->getSubprogram()) {
937 DIBuilder DB(*M, true, SP->getUnit());
938 Metadata *FunctionNameAnnotation[] = {
939 MDString::get(Ctx, InstrProfCorrelator::FunctionNameAttributeName),
940 MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)),
941 };
942 Metadata *CFGHashAnnotation[] = {
943 MDString::get(Ctx, InstrProfCorrelator::CFGHashAttributeName),
944 ConstantAsMetadata::get(Inc->getHash()),
945 };
946 Metadata *NumCountersAnnotation[] = {
947 MDString::get(Ctx, InstrProfCorrelator::NumCountersAttributeName),
948 ConstantAsMetadata::get(Inc->getNumCounters()),
949 };
950 auto Annotations = DB.getOrCreateArray({
951 MDNode::get(Ctx, FunctionNameAnnotation),
952 MDNode::get(Ctx, CFGHashAnnotation),
953 MDNode::get(Ctx, NumCountersAnnotation),
954 });
955 auto *DICounter = DB.createGlobalVariableExpression(
956 SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
957 /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),
958 CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
959 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
960 Annotations);
961 CounterPtr->addDebugInfo(DICounter);
962 DB.finalize();
963 } else {
964 std::string Msg = ("Missing debug info for function " + Fn->getName() +
965 "; required for profile correlation.")
966 .str();
967 Ctx.diagnose(
968 DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
969 }
970 }
971
972 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
973 // Allocate statically the array of pointers to value profile nodes for
974 // the current function.
975 Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
976 uint64_t NS = 0;
977 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
978 NS += PD.NumValueSites[Kind];
979 if (NS > 0 && ValueProfileStaticAlloc &&
980 !needsRuntimeRegistrationOfSectionRange(TT)) {
981 ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
982 auto *ValuesVar = new GlobalVariable(
983 *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
984 getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
985 ValuesVar->setVisibility(Visibility);
986 ValuesVar->setSection(
987 getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
988 ValuesVar->setAlignment(Align(8));
989 MaybeSetComdat(ValuesVar);
990 ValuesPtrExpr =
991 ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
992 }
993
994 if (DebugInfoCorrelate) {
995 // Mark the counter variable as used so that it isn't optimized out.
996 CompilerUsedVars.push_back(PD.RegionCounters);
997 return PD.RegionCounters;
998 }
999
1000 // Create data variable.
1001 auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext());
1002 auto *Int16Ty = Type::getInt16Ty(Ctx);
1003 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
1004 Type *DataTypes[] = {
1005#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1006#include "llvm/ProfileData/InstrProfData.inc"
1007 };
1008 auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
1009
1010 Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
1011 ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
1012 : ConstantPointerNull::get(Int8PtrTy);
1013
1014 Constant *Int16ArrayVals[IPVK_Last + 1];
1015 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1016 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
1017
1018 // If the data variable is not referenced by code (if we don't emit
1019 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
1020 // data variable live under linker GC, the data variable can be private. This
1021 // optimization applies to ELF.
1022 //
1023 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
1024 // to be false.
1025 //
1026 // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
1027 // that other copies must have the same CFG and cannot have value profiling.
1028 // If no hash suffix, other profd copies may be referenced by code.
1029 if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1030 (TT.isOSBinFormatELF() ||
1031 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
1032 Linkage = GlobalValue::PrivateLinkage;
1033 Visibility = GlobalValue::DefaultVisibility;
1034 }
1035 auto *Data =
1036 new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName);
1037 // Reference the counter variable with a label difference (link-time
1038 // constant).
1039 auto *RelativeCounterPtr =
1040 ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
1041 ConstantExpr::getPtrToInt(Data, IntPtrTy));
1042
1043 Constant *DataVals[] = {
1044#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1045#include "llvm/ProfileData/InstrProfData.inc"
1046 };
1047 Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
1048
1049 Data->setVisibility(Visibility);
1050 Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
1051 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT8));
1052 MaybeSetComdat(Data);
1053
1054 PD.DataVar = Data;
1055
1056 // Mark the data variable as used so that it isn't stripped out.
1057 CompilerUsedVars.push_back(Data);
1058 // Now that the linkage set by the FE has been passed to the data and counter
1059 // variables, reset Name variable's linkage and visibility to private so that
1060 // it can be removed later by the compiler.
1061 NamePtr->setLinkage(GlobalValue::PrivateLinkage);
1062 // Collect the referenced names to be used by emitNameData.
1063 ReferencedNames.push_back(NamePtr);
1064
1065 return PD.RegionCounters;
1066}
1067
1068void InstrProfiling::emitVNodes() {
1069 if (!ValueProfileStaticAlloc)
1070 return;
1071
1072 // For now only support this on platforms that do
1073 // not require runtime registration to discover
1074 // named section start/end.
1075 if (needsRuntimeRegistrationOfSectionRange(TT))
1076 return;
1077
1078 size_t TotalNS = 0;
1079 for (auto &PD : ProfileDataMap) {
1080 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1081 TotalNS += PD.second.NumValueSites[Kind];
1082 }
1083
1084 if (!TotalNS)
1085 return;
1086
1087 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1088// Heuristic for small programs with very few total value sites.
1089// The default value of vp-counters-per-site is chosen based on
1090// the observation that large apps usually have a low percentage
1091// of value sites that actually have any profile data, and thus
1092// the average number of counters per site is low. For small
1093// apps with very few sites, this may not be true. Bump up the
1094// number of counters in this case.
1095#define INSTR_PROF_MIN_VAL_COUNTS10 10
1096 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS10)
1097 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS10, (int)NumCounters * 2);
1098
1099 auto &Ctx = M->getContext();
1100 Type *VNodeTypes[] = {
1101#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1102#include "llvm/ProfileData/InstrProfData.inc"
1103 };
1104 auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
1105
1106 ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
1107 auto *VNodesVar = new GlobalVariable(
1108 *M, VNodesTy, false, GlobalValue::PrivateLinkage,
1109 Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
1110 VNodesVar->setSection(
1111 getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
1112 // VNodesVar is used by runtime but not referenced via relocation by other
1113 // sections. Conservatively make it linker retained.
1114 UsedVars.push_back(VNodesVar);
1115}
1116
1117void InstrProfiling::emitNameData() {
1118 std::string UncompressedData;
1119
1120 if (ReferencedNames.empty())
1121 return;
1122
1123 std::string CompressedNameStr;
1124 if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
1125 DoInstrProfNameCompression)) {
1126 report_fatal_error(Twine(toString(std::move(E))), false);
1127 }
1128
1129 auto &Ctx = M->getContext();
1130 auto *NamesVal =
1131 ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false);
1132 NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
1133 GlobalValue::PrivateLinkage, NamesVal,
1134 getInstrProfNamesVarName());
1135 NamesSize = CompressedNameStr.size();
1136 NamesVar->setSection(
1137 getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
1138 // On COFF, it's important to reduce the alignment down to 1 to prevent the
1139 // linker from inserting padding before the start of the names section or
1140 // between names entries.
1141 NamesVar->setAlignment(Align(1));
1142 // NamesVar is used by runtime but not referenced via relocation by other
1143 // sections. Conservatively make it linker retained.
1144 UsedVars.push_back(NamesVar);
1145
1146 for (auto *NamePtr : ReferencedNames)
1147 NamePtr->eraseFromParent();
1148}
1149
1150void InstrProfiling::emitRegistration() {
1151 if (!needsRuntimeRegistrationOfSectionRange(TT))
1152 return;
1153
1154 // Construct the function.
1155 auto *VoidTy = Type::getVoidTy(M->getContext());
1156 auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
1157 auto *Int64Ty = Type::getInt64Ty(M->getContext());
1158 auto *RegisterFTy = FunctionType::get(VoidTy, false);
1159 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1160 getInstrProfRegFuncsName(), M);
1161 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1162 if (Options.NoRedZone)
1163 RegisterF->addFnAttr(Attribute::NoRedZone);
1164
1165 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1166 auto *RuntimeRegisterF =
1167 Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
1168 getInstrProfRegFuncName(), M);
1169
1170 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
1171 for (Value *Data : CompilerUsedVars)
1172 if (!isa<Function>(Data))
1173 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1174 for (Value *Data : UsedVars)
1175 if (Data != NamesVar && !isa<Function>(Data))
1176 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1177
1178 if (NamesVar) {
1179 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1180 auto *NamesRegisterTy =
1181 FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
1182 auto *NamesRegisterF =
1183 Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
1184 getInstrProfNamesRegFuncName(), M);
1185 IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
1186 IRB.getInt64(NamesSize)});
1187 }
1188
1189 IRB.CreateRetVoid();
1190}
1191
1192bool InstrProfiling::emitRuntimeHook() {
1193 // We expect the linker to be invoked with -u<hook_var> flag for Linux
1194 // in which case there is no need to emit the external variable.
1195 if (TT.isOSLinux())
1196 return false;
1197
1198 // If the module's provided its own runtime, we don't need to do anything.
1199 if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
1200 return false;
1201
1202 // Declare an external variable that will pull in the runtime initialization.
1203 auto *Int32Ty = Type::getInt32Ty(M->getContext());
1204 auto *Var =
1205 new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
1206 nullptr, getInstrProfRuntimeHookVarName());
1207 Var->setVisibility(GlobalValue::HiddenVisibility);
1208
1209 if (TT.isOSBinFormatELF() && !TT.isPS()) {
1210 // Mark the user variable as used so that it isn't stripped out.
1211 CompilerUsedVars.push_back(Var);
1212 } else {
1213 // Make a function that uses it.
1214 auto *User = Function::Create(FunctionType::get(Int32Ty, false),
1215 GlobalValue::LinkOnceODRLinkage,
1216 getInstrProfRuntimeHookVarUseFuncName(), M);
1217 User->addFnAttr(Attribute::NoInline);
1218 if (Options.NoRedZone)
1219 User->addFnAttr(Attribute::NoRedZone);
1220 User->setVisibility(GlobalValue::HiddenVisibility);
1221 if (TT.supportsCOMDAT())
1222 User->setComdat(M->getOrInsertComdat(User->getName()));
1223
1224 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
1225 auto *Load = IRB.CreateLoad(Int32Ty, Var);
1226 IRB.CreateRet(Load);
1227
1228 // Mark the function as used so that it isn't stripped out.
1229 CompilerUsedVars.push_back(User);
1230 }
1231 return true;
1232}
1233
1234void InstrProfiling::emitUses() {
1235 // The metadata sections are parallel arrays. Optimizers (e.g.
1236 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
1237 // we conservatively retain all unconditionally in the compiler.
1238 //
1239 // On ELF and Mach-O, the linker can guarantee the associated sections will be
1240 // retained or discarded as a unit, so llvm.compiler.used is sufficient.
1241 // Similarly on COFF, if prof data is not referenced by code we use one comdat
1242 // and ensure this GC property as well. Otherwise, we have to conservatively
1243 // make all of the sections retained by the linker.
1244 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
1245 (TT.isOSBinFormatCOFF() && !profDataReferencedByCode(*M)))
1246 appendToCompilerUsed(*M, CompilerUsedVars);
1247 else
1248 appendToUsed(*M, CompilerUsedVars);
1249
1250 // We do not add proper references from used metadata sections to NamesVar and
1251 // VNodesVar, so we have to be conservative and place them in llvm.used
1252 // regardless of the target,
1253 appendToUsed(*M, UsedVars);
1254}
1255
1256void InstrProfiling::emitInitialization() {
1257 // Create ProfileFileName variable. Don't don't this for the
1258 // context-sensitive instrumentation lowering: This lowering is after
1259 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1260 // have already create the variable before LTO/ThinLTO linking.
1261 if (!IsCS)
1262 createProfileFileNameVar(*M, Options.InstrProfileOutput);
1263 Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1264 if (!RegisterF)
1265 return;
1266
1267 // Create the initialization function.
1268 auto *VoidTy = Type::getVoidTy(M->getContext());
1269 auto *F = Function::Create(FunctionType::get(VoidTy, false),
1270 GlobalValue::InternalLinkage,
1271 getInstrProfInitFuncName(), M);
1272 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1273 F->addFnAttr(Attribute::NoInline);
1274 if (Options.NoRedZone)
1275 F->addFnAttr(Attribute::NoRedZone);
1276
1277 // Add the basic block and the necessary calls.
1278 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1279 IRB.CreateCall(RegisterF, {});
1280 IRB.CreateRetVoid();
1281
1282 appendToGlobalCtors(*M, F, 0);
1283}