Bug Summary

File:build/source/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
Warning:line 191, column 9
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name InstrProfiling.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-17/lib/clang/17 -D _DEBUG -D _GLIBCXX_ASSERTIONS -D _GNU_SOURCE -D _LIBCPP_ENABLE_ASSERTIONS -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Transforms/Instrumentation -I /build/source/llvm/lib/Transforms/Instrumentation -I include -I /build/source/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-17/lib/clang/17/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/source/= -fcoverage-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/source/= -source-date-epoch 1683717183 -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/source/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/source/= -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2023-05-10-133810-16478-1 -x c++ /build/source/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
15#include "llvm/Transforms/Instrumentation/InstrProfiling.h"
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SmallVector.h"
18#include "llvm/ADT/StringRef.h"
19#include "llvm/ADT/Twine.h"
20#include "llvm/Analysis/BlockFrequencyInfo.h"
21#include "llvm/Analysis/BranchProbabilityInfo.h"
22#include "llvm/Analysis/LoopInfo.h"
23#include "llvm/Analysis/TargetLibraryInfo.h"
24#include "llvm/IR/Attributes.h"
25#include "llvm/IR/BasicBlock.h"
26#include "llvm/IR/Constant.h"
27#include "llvm/IR/Constants.h"
28#include "llvm/IR/DIBuilder.h"
29#include "llvm/IR/DerivedTypes.h"
30#include "llvm/IR/DiagnosticInfo.h"
31#include "llvm/IR/Dominators.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/GlobalValue.h"
34#include "llvm/IR/GlobalVariable.h"
35#include "llvm/IR/IRBuilder.h"
36#include "llvm/IR/Instruction.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/IntrinsicInst.h"
39#include "llvm/IR/Module.h"
40#include "llvm/IR/Type.h"
41#include "llvm/InitializePasses.h"
42#include "llvm/Pass.h"
43#include "llvm/ProfileData/InstrProf.h"
44#include "llvm/ProfileData/InstrProfCorrelator.h"
45#include "llvm/Support/Casting.h"
46#include "llvm/Support/CommandLine.h"
47#include "llvm/Support/Error.h"
48#include "llvm/Support/ErrorHandling.h"
49#include "llvm/TargetParser/Triple.h"
50#include "llvm/Transforms/Utils/ModuleUtils.h"
51#include "llvm/Transforms/Utils/SSAUpdater.h"
52#include <algorithm>
53#include <cassert>
54#include <cstdint>
55#include <string>
56
57using namespace llvm;
58
59#define DEBUG_TYPE"instrprof" "instrprof"
60
61namespace llvm {
62cl::opt<bool>
63 DebugInfoCorrelate("debug-info-correlate",
64 cl::desc("Use debug info to correlate profiles."),
65 cl::init(false));
66} // namespace llvm
67
68namespace {
69
70cl::opt<bool> DoHashBasedCounterSplit(
71 "hash-based-counter-split",
72 cl::desc("Rename counter variable of a comdat function based on cfg hash"),
73 cl::init(true));
74
75cl::opt<bool>
76 RuntimeCounterRelocation("runtime-counter-relocation",
77 cl::desc("Enable relocating counters at runtime."),
78 cl::init(false));
79
80cl::opt<bool> ValueProfileStaticAlloc(
81 "vp-static-alloc",
82 cl::desc("Do static counter allocation for value profiler"),
83 cl::init(true));
84
85cl::opt<double> NumCountersPerValueSite(
86 "vp-counters-per-site",
87 cl::desc("The average number of profile counters allocated "
88 "per value profiling site."),
89 // This is set to a very small value because in real programs, only
90 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
91 // For those sites with non-zero profile, the average number of targets
92 // is usually smaller than 2.
93 cl::init(1.0));
94
95cl::opt<bool> AtomicCounterUpdateAll(
96 "instrprof-atomic-counter-update-all",
97 cl::desc("Make all profile counter updates atomic (for testing only)"),
98 cl::init(false));
99
100cl::opt<bool> AtomicCounterUpdatePromoted(
101 "atomic-counter-update-promoted",
102 cl::desc("Do counter update using atomic fetch add "
103 " for promoted counters only"),
104 cl::init(false));
105
106cl::opt<bool> AtomicFirstCounter(
107 "atomic-first-counter",
108 cl::desc("Use atomic fetch add for first counter in a function (usually "
109 "the entry counter)"),
110 cl::init(false));
111
112// If the option is not specified, the default behavior about whether
113// counter promotion is done depends on how instrumentaiton lowering
114// pipeline is setup, i.e., the default value of true of this option
115// does not mean the promotion will be done by default. Explicitly
116// setting this option can override the default behavior.
117cl::opt<bool> DoCounterPromotion("do-counter-promotion",
118 cl::desc("Do counter register promotion"),
119 cl::init(false));
120cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
121 "max-counter-promotions-per-loop", cl::init(20),
122 cl::desc("Max number counter promotions per loop to avoid"
123 " increasing register pressure too much"));
124
125// A debug option
126cl::opt<int>
127 MaxNumOfPromotions("max-counter-promotions", cl::init(-1),
128 cl::desc("Max number of allowed counter promotions"));
129
130cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
131 "speculative-counter-promotion-max-exiting", cl::init(3),
132 cl::desc("The max number of exiting blocks of a loop to allow "
133 " speculative counter promotion"));
134
135cl::opt<bool> SpeculativeCounterPromotionToLoop(
136 "speculative-counter-promotion-to-loop",
137 cl::desc("When the option is false, if the target block is in a loop, "
138 "the promotion will be disallowed unless the promoted counter "
139 " update can be further/iteratively promoted into an acyclic "
140 " region."));
141
142cl::opt<bool> IterativeCounterPromotion(
143 "iterative-counter-promotion", cl::init(true),
144 cl::desc("Allow counter promotion across the whole loop nest."));
145
146cl::opt<bool> SkipRetExitBlock(
147 "skip-ret-exit-block", cl::init(true),
148 cl::desc("Suppress counter promotion if exit blocks contain ret."));
149
150///
151/// A helper class to promote one counter RMW operation in the loop
152/// into register update.
153///
154/// RWM update for the counter will be sinked out of the loop after
155/// the transformation.
156///
157class PGOCounterPromoterHelper : public LoadAndStorePromoter {
158public:
159 PGOCounterPromoterHelper(
160 Instruction *L, Instruction *S, SSAUpdater &SSA, Value *Init,
161 BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
162 ArrayRef<Instruction *> InsertPts,
163 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
164 LoopInfo &LI)
165 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
166 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
167 assert(isa<LoadInst>(L))(static_cast <bool> (isa<LoadInst>(L)) ? void (0)
: __assert_fail ("isa<LoadInst>(L)", "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp"
, 167, __extension__ __PRETTY_FUNCTION__))
;
168 assert(isa<StoreInst>(S))(static_cast <bool> (isa<StoreInst>(S)) ? void (0
) : __assert_fail ("isa<StoreInst>(S)", "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp"
, 168, __extension__ __PRETTY_FUNCTION__))
;
169 SSA.AddAvailableValue(PH, Init);
170 }
171
172 void doExtraRewritesBeforeFinalDeletion() override {
173 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
1
Assuming 'i' is not equal to 'e'
2
Loop condition is true. Entering loop body
174 BasicBlock *ExitBlock = ExitBlocks[i];
175 Instruction *InsertPos = InsertPts[i];
176 // Get LiveIn value into the ExitBlock. If there are multiple
177 // predecessors, the value is defined by a PHI node in this
178 // block.
179 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
180 Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
3
Field 'Store' is a 'CastReturnType'
181 Type *Ty = LiveInValue->getType();
182 IRBuilder<> Builder(InsertPos);
183 if (auto *AddrInst
4.1
'AddrInst' is non-null
= dyn_cast_or_null<IntToPtrInst>(Addr)) {
4
Assuming 'Addr' is a 'CastReturnType'
5
Taking true branch
184 // If isRuntimeCounterRelocationEnabled() is true then the address of
185 // the store instruction is computed with two instructions in
186 // InstrProfiling::getCounterAddress(). We need to copy those
187 // instructions to this block to compute Addr correctly.
188 // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
189 // %Addr = inttoptr i64 %BiasAdd to i64*
190 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
6
Assuming the object is not a 'CastReturnType'
7
'OrigBiasInst' initialized to a null pointer value
191 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add)(static_cast <bool> (OrigBiasInst->getOpcode() == Instruction
::BinaryOps::Add) ? void (0) : __assert_fail ("OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add"
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 191
, __extension__ __PRETTY_FUNCTION__))
;
8
Called C++ object pointer is null
192 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
193 Addr = Builder.CreateIntToPtr(BiasInst, Ty->getPointerTo());
194 }
195 if (AtomicCounterUpdatePromoted)
196 // automic update currently can only be promoted across the current
197 // loop, not the whole loop nest.
198 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
199 MaybeAlign(),
200 AtomicOrdering::SequentiallyConsistent);
201 else {
202 LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
203 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
204 auto *NewStore = Builder.CreateStore(NewVal, Addr);
205
206 // Now update the parent loop's candidate list:
207 if (IterativeCounterPromotion) {
208 auto *TargetLoop = LI.getLoopFor(ExitBlock);
209 if (TargetLoop)
210 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
211 }
212 }
213 }
214 }
215
216private:
217 Instruction *Store;
218 ArrayRef<BasicBlock *> ExitBlocks;
219 ArrayRef<Instruction *> InsertPts;
220 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
221 LoopInfo &LI;
222};
223
224/// A helper class to do register promotion for all profile counter
225/// updates in a loop.
226///
227class PGOCounterPromoter {
228public:
229 PGOCounterPromoter(
230 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCands,
231 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
232 : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {
233
234 // Skip collection of ExitBlocks and InsertPts for loops that will not be
235 // able to have counters promoted.
236 SmallVector<BasicBlock *, 8> LoopExitBlocks;
237 SmallPtrSet<BasicBlock *, 8> BlockSet;
238
239 L.getExitBlocks(LoopExitBlocks);
240 if (!isPromotionPossible(&L, LoopExitBlocks))
241 return;
242
243 for (BasicBlock *ExitBlock : LoopExitBlocks) {
244 if (BlockSet.insert(ExitBlock).second) {
245 ExitBlocks.push_back(ExitBlock);
246 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
247 }
248 }
249 }
250
251 bool run(int64_t *NumPromoted) {
252 // Skip 'infinite' loops:
253 if (ExitBlocks.size() == 0)
254 return false;
255
256 // Skip if any of the ExitBlocks contains a ret instruction.
257 // This is to prevent dumping of incomplete profile -- if the
258 // the loop is a long running loop and dump is called in the middle
259 // of the loop, the result profile is incomplete.
260 // FIXME: add other heuristics to detect long running loops.
261 if (SkipRetExitBlock) {
262 for (auto *BB : ExitBlocks)
263 if (isa<ReturnInst>(BB->getTerminator()))
264 return false;
265 }
266
267 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
268 if (MaxProm == 0)
269 return false;
270
271 unsigned Promoted = 0;
272 for (auto &Cand : LoopToCandidates[&L]) {
273
274 SmallVector<PHINode *, 4> NewPHIs;
275 SSAUpdater SSA(&NewPHIs);
276 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
277
278 // If BFI is set, we will use it to guide the promotions.
279 if (BFI) {
280 auto *BB = Cand.first->getParent();
281 auto InstrCount = BFI->getBlockProfileCount(BB);
282 if (!InstrCount)
283 continue;
284 auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
285 // If the average loop trip count is not greater than 1.5, we skip
286 // promotion.
287 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
288 continue;
289 }
290
291 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
292 L.getLoopPreheader(), ExitBlocks,
293 InsertPts, LoopToCandidates, LI);
294 Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
295 Promoted++;
296 if (Promoted >= MaxProm)
297 break;
298
299 (*NumPromoted)++;
300 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
301 break;
302 }
303
304 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("instrprof")) { dbgs() << Promoted << " counters promoted for loop (depth="
<< L.getLoopDepth() << ")\n"; } } while (false)
305 << L.getLoopDepth() << ")\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("instrprof")) { dbgs() << Promoted << " counters promoted for loop (depth="
<< L.getLoopDepth() << ")\n"; } } while (false)
;
306 return Promoted != 0;
307 }
308
309private:
310 bool allowSpeculativeCounterPromotion(Loop *LP) {
311 SmallVector<BasicBlock *, 8> ExitingBlocks;
312 L.getExitingBlocks(ExitingBlocks);
313 // Not considierered speculative.
314 if (ExitingBlocks.size() == 1)
315 return true;
316 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
317 return false;
318 return true;
319 }
320
321 // Check whether the loop satisfies the basic conditions needed to perform
322 // Counter Promotions.
323 bool
324 isPromotionPossible(Loop *LP,
325 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
326 // We can't insert into a catchswitch.
327 if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
328 return isa<CatchSwitchInst>(Exit->getTerminator());
329 }))
330 return false;
331
332 if (!LP->hasDedicatedExits())
333 return false;
334
335 BasicBlock *PH = LP->getLoopPreheader();
336 if (!PH)
337 return false;
338
339 return true;
340 }
341
342 // Returns the max number of Counter Promotions for LP.
343 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
344 SmallVector<BasicBlock *, 8> LoopExitBlocks;
345 LP->getExitBlocks(LoopExitBlocks);
346 if (!isPromotionPossible(LP, LoopExitBlocks))
347 return 0;
348
349 SmallVector<BasicBlock *, 8> ExitingBlocks;
350 LP->getExitingBlocks(ExitingBlocks);
351
352 // If BFI is set, we do more aggressive promotions based on BFI.
353 if (BFI)
354 return (unsigned)-1;
355
356 // Not considierered speculative.
357 if (ExitingBlocks.size() == 1)
358 return MaxNumOfPromotionsPerLoop;
359
360 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
361 return 0;
362
363 // Whether the target block is in a loop does not matter:
364 if (SpeculativeCounterPromotionToLoop)
365 return MaxNumOfPromotionsPerLoop;
366
367 // Now check the target block:
368 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
369 for (auto *TargetBlock : LoopExitBlocks) {
370 auto *TargetLoop = LI.getLoopFor(TargetBlock);
371 if (!TargetLoop)
372 continue;
373 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
374 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
375 MaxProm =
376 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
377 PendingCandsInTarget);
378 }
379 return MaxProm;
380 }
381
382 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> &LoopToCandidates;
383 SmallVector<BasicBlock *, 8> ExitBlocks;
384 SmallVector<Instruction *, 8> InsertPts;
385 Loop &L;
386 LoopInfo &LI;
387 BlockFrequencyInfo *BFI;
388};
389
390enum class ValueProfilingCallType {
391 // Individual values are tracked. Currently used for indiret call target
392 // profiling.
393 Default,
394
395 // MemOp: the memop size value profiling.
396 MemOp
397};
398
399} // end anonymous namespace
400
401PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) {
402 FunctionAnalysisManager &FAM =
403 AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
404 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
405 return FAM.getResult<TargetLibraryAnalysis>(F);
406 };
407 if (!run(M, GetTLI))
408 return PreservedAnalyses::all();
409
410 return PreservedAnalyses::none();
411}
412
413bool InstrProfiling::lowerIntrinsics(Function *F) {
414 bool MadeChange = false;
415 PromotionCandidates.clear();
416 for (BasicBlock &BB : *F) {
417 for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
418 if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
419 lowerIncrement(IPIS);
420 MadeChange = true;
421 } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
422 lowerIncrement(IPI);
423 MadeChange = true;
424 } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(&Instr)) {
425 lowerTimestamp(IPC);
426 MadeChange = true;
427 } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
428 lowerCover(IPC);
429 MadeChange = true;
430 } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
431 lowerValueProfileInst(IPVP);
432 MadeChange = true;
433 }
434 }
435 }
436
437 if (!MadeChange)
438 return false;
439
440 promoteCounterLoadStores(F);
441 return true;
442}
443
444bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
445 // Mach-O don't support weak external references.
446 if (TT.isOSBinFormatMachO())
447 return false;
448
449 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
450 return RuntimeCounterRelocation;
451
452 // Fuchsia uses runtime counter relocation by default.
453 return TT.isOSFuchsia();
454}
455
456bool InstrProfiling::isCounterPromotionEnabled() const {
457 if (DoCounterPromotion.getNumOccurrences() > 0)
458 return DoCounterPromotion;
459
460 return Options.DoCounterPromotion;
461}
462
463void InstrProfiling::promoteCounterLoadStores(Function *F) {
464 if (!isCounterPromotionEnabled())
465 return;
466
467 DominatorTree DT(*F);
468 LoopInfo LI(DT);
469 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
470
471 std::unique_ptr<BlockFrequencyInfo> BFI;
472 if (Options.UseBFIInPromotion) {
473 std::unique_ptr<BranchProbabilityInfo> BPI;
474 BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
475 BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
476 }
477
478 for (const auto &LoadStore : PromotionCandidates) {
479 auto *CounterLoad = LoadStore.first;
480 auto *CounterStore = LoadStore.second;
481 BasicBlock *BB = CounterLoad->getParent();
482 Loop *ParentLoop = LI.getLoopFor(BB);
483 if (!ParentLoop)
484 continue;
485 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
486 }
487
488 SmallVector<Loop *, 4> Loops = LI.getLoopsInPreorder();
489
490 // Do a post-order traversal of the loops so that counter updates can be
491 // iteratively hoisted outside the loop nest.
492 for (auto *Loop : llvm::reverse(Loops)) {
493 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
494 Promoter.run(&TotalCountersPromoted);
495 }
496}
497
498static bool needsRuntimeHookUnconditionally(const Triple &TT) {
499 // On Fuchsia, we only need runtime hook if any counters are present.
500 if (TT.isOSFuchsia())
501 return false;
502
503 return true;
504}
505
506/// Check if the module contains uses of any profiling intrinsics.
507static bool containsProfilingIntrinsics(Module &M) {
508 auto containsIntrinsic = [&](int ID) {
509 if (auto *F = M.getFunction(Intrinsic::getName(ID)))
510 return !F->use_empty();
511 return false;
512 };
513 return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
514 containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
515 containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
516 containsIntrinsic(llvm::Intrinsic::instrprof_timestamp) ||
517 containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
518}
519
520bool InstrProfiling::run(
521 Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
522 this->M = &M;
523 this->GetTLI = std::move(GetTLI);
524 NamesVar = nullptr;
525 NamesSize = 0;
526 ProfileDataMap.clear();
527 CompilerUsedVars.clear();
528 UsedVars.clear();
529 TT = Triple(M.getTargetTriple());
530
531 bool MadeChange = false;
532 bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
533 if (NeedsRuntimeHook)
534 MadeChange = emitRuntimeHook();
535
536 bool ContainsProfiling = containsProfilingIntrinsics(M);
537 GlobalVariable *CoverageNamesVar =
538 M.getNamedGlobal(getCoverageUnusedNamesVarName());
539 // Improve compile time by avoiding linear scans when there is no work.
540 if (!ContainsProfiling && !CoverageNamesVar)
541 return MadeChange;
542
543 // We did not know how many value sites there would be inside
544 // the instrumented function. This is counting the number of instrumented
545 // target value sites to enter it as field in the profile data variable.
546 for (Function &F : M) {
547 InstrProfInstBase *FirstProfInst = nullptr;
548 for (BasicBlock &BB : F)
549 for (auto I = BB.begin(), E = BB.end(); I != E; I++)
550 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
551 computeNumValueSiteCounts(Ind);
552 else if (FirstProfInst == nullptr &&
553 (isa<InstrProfIncrementInst>(I) || isa<InstrProfCoverInst>(I)))
554 FirstProfInst = dyn_cast<InstrProfInstBase>(I);
555
556 // Value profiling intrinsic lowering requires per-function profile data
557 // variable to be created first.
558 if (FirstProfInst != nullptr)
559 static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
560 }
561
562 for (Function &F : M)
563 MadeChange |= lowerIntrinsics(&F);
564
565 if (CoverageNamesVar) {
566 lowerCoverageData(CoverageNamesVar);
567 MadeChange = true;
568 }
569
570 if (!MadeChange)
571 return false;
572
573 emitVNodes();
574 emitNameData();
575
576 // Emit runtime hook for the cases where the target does not unconditionally
577 // require pulling in profile runtime, and coverage is enabled on code that is
578 // not eliminated by the front-end, e.g. unused functions with internal
579 // linkage.
580 if (!NeedsRuntimeHook && ContainsProfiling)
581 emitRuntimeHook();
582
583 emitRegistration();
584 emitUses();
585 emitInitialization();
586 return true;
587}
588
589static FunctionCallee getOrInsertValueProfilingCall(
590 Module &M, const TargetLibraryInfo &TLI,
591 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
592 LLVMContext &Ctx = M.getContext();
593 auto *ReturnTy = Type::getVoidTy(M.getContext());
594
595 AttributeList AL;
596 if (auto AK = TLI.getExtAttrForI32Param(false))
597 AL = AL.addParamAttribute(M.getContext(), 2, AK);
598
599 assert((CallType == ValueProfilingCallType::Default ||(static_cast <bool> ((CallType == ValueProfilingCallType
::Default || CallType == ValueProfilingCallType::MemOp) &&
"Must be Default or MemOp") ? void (0) : __assert_fail ("(CallType == ValueProfilingCallType::Default || CallType == ValueProfilingCallType::MemOp) && \"Must be Default or MemOp\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 601
, __extension__ __PRETTY_FUNCTION__))
600 CallType == ValueProfilingCallType::MemOp) &&(static_cast <bool> ((CallType == ValueProfilingCallType
::Default || CallType == ValueProfilingCallType::MemOp) &&
"Must be Default or MemOp") ? void (0) : __assert_fail ("(CallType == ValueProfilingCallType::Default || CallType == ValueProfilingCallType::MemOp) && \"Must be Default or MemOp\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 601
, __extension__ __PRETTY_FUNCTION__))
601 "Must be Default or MemOp")(static_cast <bool> ((CallType == ValueProfilingCallType
::Default || CallType == ValueProfilingCallType::MemOp) &&
"Must be Default or MemOp") ? void (0) : __assert_fail ("(CallType == ValueProfilingCallType::Default || CallType == ValueProfilingCallType::MemOp) && \"Must be Default or MemOp\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 601
, __extension__ __PRETTY_FUNCTION__))
;
602 Type *ParamTypes[] = {
603#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
604#include "llvm/ProfileData/InstrProfData.inc"
605 };
606 auto *ValueProfilingCallTy =
607 FunctionType::get(ReturnTy, ArrayRef(ParamTypes), false);
608 StringRef FuncName = CallType == ValueProfilingCallType::Default
609 ? getInstrProfValueProfFuncName()
610 : getInstrProfValueProfMemOpFuncName();
611 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
612}
613
614void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
615 GlobalVariable *Name = Ind->getName();
616 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
617 uint64_t Index = Ind->getIndex()->getZExtValue();
618 auto &PD = ProfileDataMap[Name];
619 PD.NumValueSites[ValueKind] =
620 std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));
621}
622
623void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
624 // TODO: Value profiling heavily depends on the data section which is omitted
625 // in lightweight mode. We need to move the value profile pointer to the
626 // Counter struct to get this working.
627 assert((static_cast <bool> (!DebugInfoCorrelate && "Value profiling is not yet supported with lightweight instrumentation"
) ? void (0) : __assert_fail ("!DebugInfoCorrelate && \"Value profiling is not yet supported with lightweight instrumentation\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 629
, __extension__ __PRETTY_FUNCTION__))
628 !DebugInfoCorrelate &&(static_cast <bool> (!DebugInfoCorrelate && "Value profiling is not yet supported with lightweight instrumentation"
) ? void (0) : __assert_fail ("!DebugInfoCorrelate && \"Value profiling is not yet supported with lightweight instrumentation\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 629
, __extension__ __PRETTY_FUNCTION__))
629 "Value profiling is not yet supported with lightweight instrumentation")(static_cast <bool> (!DebugInfoCorrelate && "Value profiling is not yet supported with lightweight instrumentation"
) ? void (0) : __assert_fail ("!DebugInfoCorrelate && \"Value profiling is not yet supported with lightweight instrumentation\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 629
, __extension__ __PRETTY_FUNCTION__))
;
630 GlobalVariable *Name = Ind->getName();
631 auto It = ProfileDataMap.find(Name);
632 assert(It != ProfileDataMap.end() && It->second.DataVar &&(static_cast <bool> (It != ProfileDataMap.end() &&
It->second.DataVar && "value profiling detected in function with no counter incerement"
) ? void (0) : __assert_fail ("It != ProfileDataMap.end() && It->second.DataVar && \"value profiling detected in function with no counter incerement\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 633
, __extension__ __PRETTY_FUNCTION__))
633 "value profiling detected in function with no counter incerement")(static_cast <bool> (It != ProfileDataMap.end() &&
It->second.DataVar && "value profiling detected in function with no counter incerement"
) ? void (0) : __assert_fail ("It != ProfileDataMap.end() && It->second.DataVar && \"value profiling detected in function with no counter incerement\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 633
, __extension__ __PRETTY_FUNCTION__))
;
634
635 GlobalVariable *DataVar = It->second.DataVar;
636 uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
637 uint64_t Index = Ind->getIndex()->getZExtValue();
638 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
639 Index += It->second.NumValueSites[Kind];
640
641 IRBuilder<> Builder(Ind);
642 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
643 llvm::InstrProfValueKind::IPVK_MemOPSize);
644 CallInst *Call = nullptr;
645 auto *TLI = &GetTLI(*Ind->getFunction());
646
647 // To support value profiling calls within Windows exception handlers, funclet
648 // information contained within operand bundles needs to be copied over to
649 // the library call. This is required for the IR to be processed by the
650 // WinEHPrepare pass.
651 SmallVector<OperandBundleDef, 1> OpBundles;
652 Ind->getOperandBundlesAsDefs(OpBundles);
653 if (!IsMemOpSize) {
654 Value *Args[3] = {Ind->getTargetValue(),
655 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
656 Builder.getInt32(Index)};
657 Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
658 OpBundles);
659 } else {
660 Value *Args[3] = {Ind->getTargetValue(),
661 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
662 Builder.getInt32(Index)};
663 Call = Builder.CreateCall(
664 getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
665 Args, OpBundles);
666 }
667 if (auto AK = TLI->getExtAttrForI32Param(false))
668 Call->addParamAttr(2, AK);
669 Ind->replaceAllUsesWith(Call);
670 Ind->eraseFromParent();
671}
672
673Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) {
674 auto *Counters = getOrCreateRegionCounters(I);
675 IRBuilder<> Builder(I);
676
677 if (isa<InstrProfTimestampInst>(I))
678 Counters->setAlignment(Align(8));
679
680 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
681 Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());
682
683 if (!isRuntimeCounterRelocationEnabled())
684 return Addr;
685
686 Type *Int64Ty = Type::getInt64Ty(M->getContext());
687 Function *Fn = I->getParent()->getParent();
688 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
689 if (!BiasLI) {
690 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
691 auto *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
692 if (!Bias) {
693 // Compiler must define this variable when runtime counter relocation
694 // is being used. Runtime has a weak external reference that is used
695 // to check whether that's the case or not.
696 Bias = new GlobalVariable(
697 *M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
698 Constant::getNullValue(Int64Ty), getInstrProfCounterBiasVarName());
699 Bias->setVisibility(GlobalVariable::HiddenVisibility);
700 // A definition that's weak (linkonce_odr) without being in a COMDAT
701 // section wouldn't lead to link errors, but it would lead to a dead
702 // data word from every TU but one. Putting it in COMDAT ensures there
703 // will be exactly one data slot in the link.
704 if (TT.supportsCOMDAT())
705 Bias->setComdat(M->getOrInsertComdat(Bias->getName()));
706 }
707 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
708 }
709 auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
710 return Builder.CreateIntToPtr(Add, Addr->getType());
711}
712
713void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) {
714 auto *Addr = getCounterAddress(CoverInstruction);
715 IRBuilder<> Builder(CoverInstruction);
716 // We store zero to represent that this block is covered.
717 Builder.CreateStore(Builder.getInt8(0), Addr);
718 CoverInstruction->eraseFromParent();
719}
720
721void InstrProfiling::lowerTimestamp(
722 InstrProfTimestampInst *TimestampInstruction) {
723 assert(TimestampInstruction->getIndex()->isZeroValue() &&(static_cast <bool> (TimestampInstruction->getIndex(
)->isZeroValue() && "timestamp probes are always the first probe for a function"
) ? void (0) : __assert_fail ("TimestampInstruction->getIndex()->isZeroValue() && \"timestamp probes are always the first probe for a function\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 724
, __extension__ __PRETTY_FUNCTION__))
724 "timestamp probes are always the first probe for a function")(static_cast <bool> (TimestampInstruction->getIndex(
)->isZeroValue() && "timestamp probes are always the first probe for a function"
) ? void (0) : __assert_fail ("TimestampInstruction->getIndex()->isZeroValue() && \"timestamp probes are always the first probe for a function\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 724
, __extension__ __PRETTY_FUNCTION__))
;
725 auto &Ctx = M->getContext();
726 auto *TimestampAddr = getCounterAddress(TimestampInstruction);
727 IRBuilder<> Builder(TimestampInstruction);
728 auto *CalleeTy =
729 FunctionType::get(Type::getVoidTy(Ctx), TimestampAddr->getType(), false);
730 auto Callee = M->getOrInsertFunction(
731 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP)"__llvm_profile_set_timestamp", CalleeTy);
732 Builder.CreateCall(Callee, {TimestampAddr});
733 TimestampInstruction->eraseFromParent();
734}
735
736void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
737 auto *Addr = getCounterAddress(Inc);
738
739 IRBuilder<> Builder(Inc);
740 if (Options.Atomic || AtomicCounterUpdateAll ||
741 (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {
742 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
743 MaybeAlign(), AtomicOrdering::Monotonic);
744 } else {
745 Value *IncStep = Inc->getStep();
746 Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
747 auto *Count = Builder.CreateAdd(Load, Inc->getStep());
748 auto *Store = Builder.CreateStore(Count, Addr);
749 if (isCounterPromotionEnabled())
750 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
751 }
752 Inc->eraseFromParent();
753}
754
755void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
756 ConstantArray *Names =
757 cast<ConstantArray>(CoverageNamesVar->getInitializer());
758 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
759 Constant *NC = Names->getOperand(I);
760 Value *V = NC->stripPointerCasts();
761 assert(isa<GlobalVariable>(V) && "Missing reference to function name")(static_cast <bool> (isa<GlobalVariable>(V) &&
"Missing reference to function name") ? void (0) : __assert_fail
("isa<GlobalVariable>(V) && \"Missing reference to function name\""
, "llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp", 761
, __extension__ __PRETTY_FUNCTION__))
;
762 GlobalVariable *Name = cast<GlobalVariable>(V);
763
764 Name->setLinkage(GlobalValue::PrivateLinkage);
765 ReferencedNames.push_back(Name);
766 if (isa<ConstantExpr>(NC))
767 NC->dropAllReferences();
768 }
769 CoverageNamesVar->eraseFromParent();
770}
771
772/// Get the name of a profiling variable for a particular function.
773static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
774 bool &Renamed) {
775 StringRef NamePrefix = getInstrProfNameVarPrefix();
776 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
777 Function *F = Inc->getParent()->getParent();
778 Module *M = F->getParent();
779 if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
780 !canRenameComdatFunc(*F)) {
781 Renamed = false;
782 return (Prefix + Name).str();
783 }
784 Renamed = true;
785 uint64_t FuncHash = Inc->getHash()->getZExtValue();
786 SmallVector<char, 24> HashPostfix;
787 if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
788 return (Prefix + Name).str();
789 return (Prefix + Name + "." + Twine(FuncHash)).str();
790}
791
792static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {
793 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
794 if (!MD)
795 return 0;
796
797 // If the flag is a ConstantAsMetadata, it should be an integer representable
798 // in 64-bits.
799 return cast<ConstantInt>(MD->getValue())->getZExtValue();
800}
801
802static bool enablesValueProfiling(const Module &M) {
803 return isIRPGOFlagSet(&M) ||
804 getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;
805}
806
807// Conservatively returns true if data variables may be referenced by code.
808static bool profDataReferencedByCode(const Module &M) {
809 return enablesValueProfiling(M);
810}
811
812static inline bool shouldRecordFunctionAddr(Function *F) {
813 // Only record function addresses if IR PGO is enabled or if clang value
814 // profiling is enabled. Recording function addresses greatly increases object
815 // file size, because it prevents the inliner from deleting functions that
816 // have been inlined everywhere.
817 if (!profDataReferencedByCode(*F->getParent()))
818 return false;
819
820 // Check the linkage
821 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
822 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
823 !HasAvailableExternallyLinkage)
824 return true;
825
826 // A function marked 'alwaysinline' with available_externally linkage can't
827 // have its address taken. Doing so would create an undefined external ref to
828 // the function, which would fail to link.
829 if (HasAvailableExternallyLinkage &&
830 F->hasFnAttribute(Attribute::AlwaysInline))
831 return false;
832
833 // Prohibit function address recording if the function is both internal and
834 // COMDAT. This avoids the profile data variable referencing internal symbols
835 // in COMDAT.
836 if (F->hasLocalLinkage() && F->hasComdat())
837 return false;
838
839 // Check uses of this function for other than direct calls or invokes to it.
840 // Inline virtual functions have linkeOnceODR linkage. When a key method
841 // exists, the vtable will only be emitted in the TU where the key method
842 // is defined. In a TU where vtable is not available, the function won't
843 // be 'addresstaken'. If its address is not recorded here, the profile data
844 // with missing address may be picked by the linker leading to missing
845 // indirect call target info.
846 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
847}
848
849static inline bool shouldUsePublicSymbol(Function *Fn) {
850 // It isn't legal to make an alias of this function at all
851 if (Fn->isDeclarationForLinker())
852 return true;
853
854 // Symbols with local linkage can just use the symbol directly without
855 // introducing relocations
856 if (Fn->hasLocalLinkage())
857 return true;
858
859 // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some
860 // unfavorable interaction between the new alias and the alias renaming done
861 // in LowerTypeTests under ThinLTO. For comdat functions that would normally
862 // be deduplicated, but the renaming scheme ends up preventing renaming, since
863 // it creates unique names for each alias, resulting in duplicated symbols. In
864 // the future, we should update the CFI related passes to migrate these
865 // aliases to the same module as the jump-table they refer to will be defined.
866 if (Fn->hasMetadata(LLVMContext::MD_type))
867 return true;
868
869 // For comdat functions, an alias would need the same linkage as the original
870 // function and hidden visibility. There is no point in adding an alias with
871 // identical linkage an visibility to avoid introducing symbolic relocations.
872 if (Fn->hasComdat() &&
873 (Fn->getVisibility() == GlobalValue::VisibilityTypes::HiddenVisibility))
874 return true;
875
876 // its OK to use an alias
877 return false;
878}
879
880static inline Constant *getFuncAddrForProfData(Function *Fn) {
881 auto *Int8PtrTy = Type::getInt8PtrTy(Fn->getContext());
882 // Store a nullptr in __llvm_profd, if we shouldn't use a real address
883 if (!shouldRecordFunctionAddr(Fn))
884 return ConstantPointerNull::get(Int8PtrTy);
885
886 // If we can't use an alias, we must use the public symbol, even though this
887 // may require a symbolic relocation.
888 if (shouldUsePublicSymbol(Fn))
889 return ConstantExpr::getBitCast(Fn, Int8PtrTy);
890
891 // When possible use a private alias to avoid symbolic relocations.
892 auto *GA = GlobalAlias::create(GlobalValue::LinkageTypes::PrivateLinkage,
893 Fn->getName() + ".local", Fn);
894
895 // When the instrumented function is a COMDAT function, we cannot use a
896 // private alias. If we did, we would create reference to a local label in
897 // this function's section. If this version of the function isn't selected by
898 // the linker, then the metadata would introduce a reference to a discarded
899 // section. So, for COMDAT functions, we need to adjust the linkage of the
900 // alias. Using hidden visibility avoids a dynamic relocation and an entry in
901 // the dynamic symbol table.
902 //
903 // Note that this handles COMDAT functions with visibility other than Hidden,
904 // since that case is covered in shouldUsePublicSymbol()
905 if (Fn->hasComdat()) {
906 GA->setLinkage(Fn->getLinkage());
907 GA->setVisibility(GlobalValue::VisibilityTypes::HiddenVisibility);
908 }
909
910 // appendToCompilerUsed(*Fn->getParent(), {GA});
911
912 return ConstantExpr::getBitCast(GA, Int8PtrTy);
913}
914
915static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) {
916 // Don't do this for Darwin. compiler-rt uses linker magic.
917 if (TT.isOSDarwin())
918 return false;
919 // Use linker script magic to get data/cnts/name start/end.
920 if (TT.isOSAIX() || TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
921 TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS() || TT.isOSWindows())
922 return false;
923
924 return true;
925}
926
927GlobalVariable *
928InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name,
929 GlobalValue::LinkageTypes Linkage) {
930 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
931 auto &Ctx = M->getContext();
932 GlobalVariable *GV;
933 if (isa<InstrProfCoverInst>(Inc)) {
934 auto *CounterTy = Type::getInt8Ty(Ctx);
935 auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);
936 // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
937 std::vector<Constant *> InitialValues(NumCounters,
938 Constant::getAllOnesValue(CounterTy));
939 GV = new GlobalVariable(*M, CounterArrTy, false, Linkage,
940 ConstantArray::get(CounterArrTy, InitialValues),
941 Name);
942 GV->setAlignment(Align(1));
943 } else {
944 auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
945 GV = new GlobalVariable(*M, CounterTy, false, Linkage,
946 Constant::getNullValue(CounterTy), Name);
947 GV->setAlignment(Align(8));
948 }
949 return GV;
950}
951
952GlobalVariable *
953InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
954 GlobalVariable *NamePtr = Inc->getName();
955 auto &PD = ProfileDataMap[NamePtr];
956 if (PD.RegionCounters)
957 return PD.RegionCounters;
958
959 // Match the linkage and visibility of the name global.
960 Function *Fn = Inc->getParent()->getParent();
961 GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
962 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
963
964 // Use internal rather than private linkage so the counter variable shows up
965 // in the symbol table when using debug info for correlation.
966 if (DebugInfoCorrelate && TT.isOSBinFormatMachO() &&
967 Linkage == GlobalValue::PrivateLinkage)
968 Linkage = GlobalValue::InternalLinkage;
969
970 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
971 // symbols in the same csect won't be discarded. When there are duplicate weak
972 // symbols, we can NOT guarantee that the relocations get resolved to the
973 // intended weak symbol, so we can not ensure the correctness of the relative
974 // CounterPtr, so we have to use private linkage for counter and data symbols.
975 if (TT.isOSBinFormatXCOFF()) {
976 Linkage = GlobalValue::PrivateLinkage;
977 Visibility = GlobalValue::DefaultVisibility;
978 }
979 // Move the name variable to the right section. Place them in a COMDAT group
980 // if the associated function is a COMDAT. This will make sure that only one
981 // copy of counters of the COMDAT function will be emitted after linking. Keep
982 // in mind that this pass may run before the inliner, so we need to create a
983 // new comdat group for the counters and profiling data. If we use the comdat
984 // of the parent function, that will result in relocations against discarded
985 // sections.
986 //
987 // If the data variable is referenced by code, counters and data have to be
988 // in different comdats for COFF because the Visual C++ linker will report
989 // duplicate symbol errors if there are multiple external symbols with the
990 // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
991 //
992 // For ELF, when not using COMDAT, put counters, data and values into a
993 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
994 // allows -z start-stop-gc to discard the entire group when the function is
995 // discarded.
996 bool DataReferencedByCode = profDataReferencedByCode(*M);
997 bool NeedComdat = needsComdatForCounter(*Fn, *M);
998 bool Renamed;
999 std::string CntsVarName =
1000 getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed);
1001 std::string DataVarName =
1002 getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
1003 auto MaybeSetComdat = [&](GlobalVariable *GV) {
1004 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
1005 if (UseComdat) {
1006 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
1007 ? GV->getName()
1008 : CntsVarName;
1009 Comdat *C = M->getOrInsertComdat(GroupName);
1010 if (!NeedComdat)
1011 C->setSelectionKind(Comdat::NoDeduplicate);
1012 GV->setComdat(C);
1013 // COFF doesn't allow the comdat group leader to have private linkage, so
1014 // upgrade private linkage to internal linkage to produce a symbol table
1015 // entry.
1016 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
1017 GV->setLinkage(GlobalValue::InternalLinkage);
1018 }
1019 };
1020
1021 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1022 LLVMContext &Ctx = M->getContext();
1023
1024 auto *CounterPtr = createRegionCounters(Inc, CntsVarName, Linkage);
1025 CounterPtr->setVisibility(Visibility);
1026 CounterPtr->setSection(
1027 getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
1028 CounterPtr->setLinkage(Linkage);
1029 MaybeSetComdat(CounterPtr);
1030 PD.RegionCounters = CounterPtr;
1031 if (DebugInfoCorrelate) {
1032 if (auto *SP = Fn->getSubprogram()) {
1033 DIBuilder DB(*M, true, SP->getUnit());
1034 Metadata *FunctionNameAnnotation[] = {
1035 MDString::get(Ctx, InstrProfCorrelator::FunctionNameAttributeName),
1036 MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)),
1037 };
1038 Metadata *CFGHashAnnotation[] = {
1039 MDString::get(Ctx, InstrProfCorrelator::CFGHashAttributeName),
1040 ConstantAsMetadata::get(Inc->getHash()),
1041 };
1042 Metadata *NumCountersAnnotation[] = {
1043 MDString::get(Ctx, InstrProfCorrelator::NumCountersAttributeName),
1044 ConstantAsMetadata::get(Inc->getNumCounters()),
1045 };
1046 auto Annotations = DB.getOrCreateArray({
1047 MDNode::get(Ctx, FunctionNameAnnotation),
1048 MDNode::get(Ctx, CFGHashAnnotation),
1049 MDNode::get(Ctx, NumCountersAnnotation),
1050 });
1051 auto *DICounter = DB.createGlobalVariableExpression(
1052 SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
1053 /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),
1054 CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
1055 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1056 Annotations);
1057 CounterPtr->addDebugInfo(DICounter);
1058 DB.finalize();
1059 } else {
1060 std::string Msg = ("Missing debug info for function " + Fn->getName() +
1061 "; required for profile correlation.")
1062 .str();
1063 Ctx.diagnose(
1064 DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
1065 }
1066 }
1067
1068 auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
1069 // Allocate statically the array of pointers to value profile nodes for
1070 // the current function.
1071 Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
1072 uint64_t NS = 0;
1073 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1074 NS += PD.NumValueSites[Kind];
1075 if (NS > 0 && ValueProfileStaticAlloc &&
1076 !needsRuntimeRegistrationOfSectionRange(TT)) {
1077 ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
1078 auto *ValuesVar = new GlobalVariable(
1079 *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
1080 getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
1081 ValuesVar->setVisibility(Visibility);
1082 ValuesVar->setSection(
1083 getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
1084 ValuesVar->setAlignment(Align(8));
1085 MaybeSetComdat(ValuesVar);
1086 ValuesPtrExpr =
1087 ConstantExpr::getBitCast(ValuesVar, Type::getInt8PtrTy(Ctx));
1088 }
1089
1090 if (DebugInfoCorrelate) {
1091 // Mark the counter variable as used so that it isn't optimized out.
1092 CompilerUsedVars.push_back(PD.RegionCounters);
1093 return PD.RegionCounters;
1094 }
1095
1096 // Create data variable.
1097 auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext());
1098 auto *Int16Ty = Type::getInt16Ty(Ctx);
1099 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
1100 Type *DataTypes[] = {
1101#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1102#include "llvm/ProfileData/InstrProfData.inc"
1103 };
1104 auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));
1105
1106 Constant *FunctionAddr = getFuncAddrForProfData(Fn);
1107
1108 Constant *Int16ArrayVals[IPVK_Last + 1];
1109 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1110 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
1111
1112 // If the data variable is not referenced by code (if we don't emit
1113 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
1114 // data variable live under linker GC, the data variable can be private. This
1115 // optimization applies to ELF.
1116 //
1117 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
1118 // to be false.
1119 //
1120 // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
1121 // that other copies must have the same CFG and cannot have value profiling.
1122 // If no hash suffix, other profd copies may be referenced by code.
1123 if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1124 (TT.isOSBinFormatELF() ||
1125 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
1126 Linkage = GlobalValue::PrivateLinkage;
1127 Visibility = GlobalValue::DefaultVisibility;
1128 }
1129 auto *Data =
1130 new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName);
1131 // Reference the counter variable with a label difference (link-time
1132 // constant).
1133 auto *RelativeCounterPtr =
1134 ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
1135 ConstantExpr::getPtrToInt(Data, IntPtrTy));
1136
1137 Constant *DataVals[] = {
1138#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1139#include "llvm/ProfileData/InstrProfData.inc"
1140 };
1141 Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
1142
1143 Data->setVisibility(Visibility);
1144 Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
1145 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT8));
1146 MaybeSetComdat(Data);
1147
1148 PD.DataVar = Data;
1149
1150 // Mark the data variable as used so that it isn't stripped out.
1151 CompilerUsedVars.push_back(Data);
1152 // Now that the linkage set by the FE has been passed to the data and counter
1153 // variables, reset Name variable's linkage and visibility to private so that
1154 // it can be removed later by the compiler.
1155 NamePtr->setLinkage(GlobalValue::PrivateLinkage);
1156 // Collect the referenced names to be used by emitNameData.
1157 ReferencedNames.push_back(NamePtr);
1158
1159 return PD.RegionCounters;
1160}
1161
1162void InstrProfiling::emitVNodes() {
1163 if (!ValueProfileStaticAlloc)
1164 return;
1165
1166 // For now only support this on platforms that do
1167 // not require runtime registration to discover
1168 // named section start/end.
1169 if (needsRuntimeRegistrationOfSectionRange(TT))
1170 return;
1171
1172 size_t TotalNS = 0;
1173 for (auto &PD : ProfileDataMap) {
1174 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1175 TotalNS += PD.second.NumValueSites[Kind];
1176 }
1177
1178 if (!TotalNS)
1179 return;
1180
1181 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1182// Heuristic for small programs with very few total value sites.
1183// The default value of vp-counters-per-site is chosen based on
1184// the observation that large apps usually have a low percentage
1185// of value sites that actually have any profile data, and thus
1186// the average number of counters per site is low. For small
1187// apps with very few sites, this may not be true. Bump up the
1188// number of counters in this case.
1189#define INSTR_PROF_MIN_VAL_COUNTS10 10
1190 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS10)
1191 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS10, (int)NumCounters * 2);
1192
1193 auto &Ctx = M->getContext();
1194 Type *VNodeTypes[] = {
1195#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1196#include "llvm/ProfileData/InstrProfData.inc"
1197 };
1198 auto *VNodeTy = StructType::get(Ctx, ArrayRef(VNodeTypes));
1199
1200 ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
1201 auto *VNodesVar = new GlobalVariable(
1202 *M, VNodesTy, false, GlobalValue::PrivateLinkage,
1203 Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName());
1204 VNodesVar->setSection(
1205 getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
1206 VNodesVar->setAlignment(M->getDataLayout().getABITypeAlign(VNodesTy));
1207 // VNodesVar is used by runtime but not referenced via relocation by other
1208 // sections. Conservatively make it linker retained.
1209 UsedVars.push_back(VNodesVar);
1210}
1211
1212void InstrProfiling::emitNameData() {
1213 std::string UncompressedData;
1214
1215 if (ReferencedNames.empty())
1216 return;
1217
1218 std::string CompressedNameStr;
1219 if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
1220 DoInstrProfNameCompression)) {
1221 report_fatal_error(Twine(toString(std::move(E))), false);
1222 }
1223
1224 auto &Ctx = M->getContext();
1225 auto *NamesVal =
1226 ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false);
1227 NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
1228 GlobalValue::PrivateLinkage, NamesVal,
1229 getInstrProfNamesVarName());
1230 NamesSize = CompressedNameStr.size();
1231 NamesVar->setSection(
1232 getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
1233 // On COFF, it's important to reduce the alignment down to 1 to prevent the
1234 // linker from inserting padding before the start of the names section or
1235 // between names entries.
1236 NamesVar->setAlignment(Align(1));
1237 // NamesVar is used by runtime but not referenced via relocation by other
1238 // sections. Conservatively make it linker retained.
1239 UsedVars.push_back(NamesVar);
1240
1241 for (auto *NamePtr : ReferencedNames)
1242 NamePtr->eraseFromParent();
1243}
1244
1245void InstrProfiling::emitRegistration() {
1246 if (!needsRuntimeRegistrationOfSectionRange(TT))
1247 return;
1248
1249 // Construct the function.
1250 auto *VoidTy = Type::getVoidTy(M->getContext());
1251 auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
1252 auto *Int64Ty = Type::getInt64Ty(M->getContext());
1253 auto *RegisterFTy = FunctionType::get(VoidTy, false);
1254 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1255 getInstrProfRegFuncsName(), M);
1256 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1257 if (Options.NoRedZone)
1258 RegisterF->addFnAttr(Attribute::NoRedZone);
1259
1260 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1261 auto *RuntimeRegisterF =
1262 Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage,
1263 getInstrProfRegFuncName(), M);
1264
1265 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
1266 for (Value *Data : CompilerUsedVars)
1267 if (!isa<Function>(Data))
1268 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1269 for (Value *Data : UsedVars)
1270 if (Data != NamesVar && !isa<Function>(Data))
1271 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1272
1273 if (NamesVar) {
1274 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1275 auto *NamesRegisterTy =
1276 FunctionType::get(VoidTy, ArrayRef(ParamTypes), false);
1277 auto *NamesRegisterF =
1278 Function::Create(NamesRegisterTy, GlobalVariable::ExternalLinkage,
1279 getInstrProfNamesRegFuncName(), M);
1280 IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
1281 IRB.getInt64(NamesSize)});
1282 }
1283
1284 IRB.CreateRetVoid();
1285}
1286
1287bool InstrProfiling::emitRuntimeHook() {
1288 // We expect the linker to be invoked with -u<hook_var> flag for Linux
1289 // in which case there is no need to emit the external variable.
1290 if (TT.isOSLinux() || TT.isOSAIX())
1291 return false;
1292
1293 // If the module's provided its own runtime, we don't need to do anything.
1294 if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
1295 return false;
1296
1297 // Declare an external variable that will pull in the runtime initialization.
1298 auto *Int32Ty = Type::getInt32Ty(M->getContext());
1299 auto *Var =
1300 new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage,
1301 nullptr, getInstrProfRuntimeHookVarName());
1302 Var->setVisibility(GlobalValue::HiddenVisibility);
1303
1304 if (TT.isOSBinFormatELF() && !TT.isPS()) {
1305 // Mark the user variable as used so that it isn't stripped out.
1306 CompilerUsedVars.push_back(Var);
1307 } else {
1308 // Make a function that uses it.
1309 auto *User = Function::Create(FunctionType::get(Int32Ty, false),
1310 GlobalValue::LinkOnceODRLinkage,
1311 getInstrProfRuntimeHookVarUseFuncName(), M);
1312 User->addFnAttr(Attribute::NoInline);
1313 if (Options.NoRedZone)
1314 User->addFnAttr(Attribute::NoRedZone);
1315 User->setVisibility(GlobalValue::HiddenVisibility);
1316 if (TT.supportsCOMDAT())
1317 User->setComdat(M->getOrInsertComdat(User->getName()));
1318
1319 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
1320 auto *Load = IRB.CreateLoad(Int32Ty, Var);
1321 IRB.CreateRet(Load);
1322
1323 // Mark the function as used so that it isn't stripped out.
1324 CompilerUsedVars.push_back(User);
1325 }
1326 return true;
1327}
1328
1329void InstrProfiling::emitUses() {
1330 // The metadata sections are parallel arrays. Optimizers (e.g.
1331 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
1332 // we conservatively retain all unconditionally in the compiler.
1333 //
1334 // On ELF and Mach-O, the linker can guarantee the associated sections will be
1335 // retained or discarded as a unit, so llvm.compiler.used is sufficient.
1336 // Similarly on COFF, if prof data is not referenced by code we use one comdat
1337 // and ensure this GC property as well. Otherwise, we have to conservatively
1338 // make all of the sections retained by the linker.
1339 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
1340 (TT.isOSBinFormatCOFF() && !profDataReferencedByCode(*M)))
1341 appendToCompilerUsed(*M, CompilerUsedVars);
1342 else
1343 appendToUsed(*M, CompilerUsedVars);
1344
1345 // We do not add proper references from used metadata sections to NamesVar and
1346 // VNodesVar, so we have to be conservative and place them in llvm.used
1347 // regardless of the target,
1348 appendToUsed(*M, UsedVars);
1349}
1350
1351void InstrProfiling::emitInitialization() {
1352 // Create ProfileFileName variable. Don't don't this for the
1353 // context-sensitive instrumentation lowering: This lowering is after
1354 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1355 // have already create the variable before LTO/ThinLTO linking.
1356 if (!IsCS)
1357 createProfileFileNameVar(*M, Options.InstrProfileOutput);
1358 Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1359 if (!RegisterF)
1360 return;
1361
1362 // Create the initialization function.
1363 auto *VoidTy = Type::getVoidTy(M->getContext());
1364 auto *F = Function::Create(FunctionType::get(VoidTy, false),
1365 GlobalValue::InternalLinkage,
1366 getInstrProfInitFuncName(), M);
1367 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1368 F->addFnAttr(Attribute::NoInline);
1369 if (Options.NoRedZone)
1370 F->addFnAttr(Attribute::NoRedZone);
1371
1372 // Add the basic block and the necessary calls.
1373 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1374 IRB.CreateCall(RegisterF, {});
1375 IRB.CreateRetVoid();
1376
1377 appendToGlobalCtors(*M, F, 0);
1378}