LLVM 19.0.0git
InstrProfiling.cpp
Go to the documentation of this file.
1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by an instrumentor.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/Twine.h"
25#include "llvm/IR/Attributes.h"
26#include "llvm/IR/BasicBlock.h"
27#include "llvm/IR/CFG.h"
28#include "llvm/IR/Constant.h"
29#include "llvm/IR/Constants.h"
30#include "llvm/IR/DIBuilder.h"
33#include "llvm/IR/Dominators.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/GlobalValue.h"
37#include "llvm/IR/IRBuilder.h"
38#include "llvm/IR/Instruction.h"
41#include "llvm/IR/Module.h"
42#include "llvm/IR/Type.h"
44#include "llvm/Pass.h"
49#include "llvm/Support/Error.h"
57#include <algorithm>
58#include <cassert>
59#include <cstdint>
60#include <string>
61
62using namespace llvm;
63
64#define DEBUG_TYPE "instrprof"
65
66namespace llvm {
67// Command line option to enable vtable value profiling. Defined in
68// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
70// TODO: Remove -debug-info-correlate in next LLVM release, in favor of
71// -profile-correlate=debug-info.
73 "debug-info-correlate",
74 cl::desc("Use debug info to correlate profiles. (Deprecated, use "
75 "-profile-correlate=debug-info)"),
76 cl::init(false));
77
79 "profile-correlate",
80 cl::desc("Use debug info or binary file to correlate profiles."),
83 "No profile correlation"),
85 "Use debug info to correlate"),
87 "Use binary to correlate")));
88} // namespace llvm
89
90namespace {
91
92cl::opt<bool> DoHashBasedCounterSplit(
93 "hash-based-counter-split",
94 cl::desc("Rename counter variable of a comdat function based on cfg hash"),
95 cl::init(true));
96
98 RuntimeCounterRelocation("runtime-counter-relocation",
99 cl::desc("Enable relocating counters at runtime."),
100 cl::init(false));
101
102cl::opt<bool> ValueProfileStaticAlloc(
103 "vp-static-alloc",
104 cl::desc("Do static counter allocation for value profiler"),
105 cl::init(true));
106
107cl::opt<double> NumCountersPerValueSite(
108 "vp-counters-per-site",
109 cl::desc("The average number of profile counters allocated "
110 "per value profiling site."),
111 // This is set to a very small value because in real programs, only
112 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
113 // For those sites with non-zero profile, the average number of targets
114 // is usually smaller than 2.
115 cl::init(1.0));
116
117cl::opt<bool> AtomicCounterUpdateAll(
118 "instrprof-atomic-counter-update-all",
119 cl::desc("Make all profile counter updates atomic (for testing only)"),
120 cl::init(false));
121
122cl::opt<bool> AtomicCounterUpdatePromoted(
123 "atomic-counter-update-promoted",
124 cl::desc("Do counter update using atomic fetch add "
125 " for promoted counters only"),
126 cl::init(false));
127
128cl::opt<bool> AtomicFirstCounter(
129 "atomic-first-counter",
130 cl::desc("Use atomic fetch add for first counter in a function (usually "
131 "the entry counter)"),
132 cl::init(false));
133
134// If the option is not specified, the default behavior about whether
135// counter promotion is done depends on how instrumentaiton lowering
136// pipeline is setup, i.e., the default value of true of this option
137// does not mean the promotion will be done by default. Explicitly
138// setting this option can override the default behavior.
139cl::opt<bool> DoCounterPromotion("do-counter-promotion",
140 cl::desc("Do counter register promotion"),
141 cl::init(false));
142cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
143 "max-counter-promotions-per-loop", cl::init(20),
144 cl::desc("Max number counter promotions per loop to avoid"
145 " increasing register pressure too much"));
146
147// A debug option
149 MaxNumOfPromotions("max-counter-promotions", cl::init(-1),
150 cl::desc("Max number of allowed counter promotions"));
151
152cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
153 "speculative-counter-promotion-max-exiting", cl::init(3),
154 cl::desc("The max number of exiting blocks of a loop to allow "
155 " speculative counter promotion"));
156
157cl::opt<bool> SpeculativeCounterPromotionToLoop(
158 "speculative-counter-promotion-to-loop",
159 cl::desc("When the option is false, if the target block is in a loop, "
160 "the promotion will be disallowed unless the promoted counter "
161 " update can be further/iteratively promoted into an acyclic "
162 " region."));
163
164cl::opt<bool> IterativeCounterPromotion(
165 "iterative-counter-promotion", cl::init(true),
166 cl::desc("Allow counter promotion across the whole loop nest."));
167
168cl::opt<bool> SkipRetExitBlock(
169 "skip-ret-exit-block", cl::init(true),
170 cl::desc("Suppress counter promotion if exit blocks contain ret."));
171
172using LoadStorePair = std::pair<Instruction *, Instruction *>;
173
174static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {
175 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
176 if (!MD)
177 return 0;
178
179 // If the flag is a ConstantAsMetadata, it should be an integer representable
180 // in 64-bits.
181 return cast<ConstantInt>(MD->getValue())->getZExtValue();
182}
183
184static bool enablesValueProfiling(const Module &M) {
185 return isIRPGOFlagSet(&M) ||
186 getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;
187}
188
189// Conservatively returns true if value profiling is enabled.
190static bool profDataReferencedByCode(const Module &M) {
191 return enablesValueProfiling(M);
192}
193
194class InstrLowerer final {
195public:
196 InstrLowerer(Module &M, const InstrProfOptions &Options,
197 std::function<const TargetLibraryInfo &(Function &F)> GetTLI,
198 bool IsCS)
199 : M(M), Options(Options), TT(Triple(M.getTargetTriple())), IsCS(IsCS),
200 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {}
201
202 bool lower();
203
204private:
205 Module &M;
207 const Triple TT;
208 // Is this lowering for the context-sensitive instrumentation.
209 const bool IsCS;
210
211 std::function<const TargetLibraryInfo &(Function &F)> GetTLI;
212
213 const bool DataReferencedByCode;
214
215 struct PerFunctionProfileData {
216 uint32_t NumValueSites[IPVK_Last + 1] = {};
217 GlobalVariable *RegionCounters = nullptr;
218 GlobalVariable *DataVar = nullptr;
219 GlobalVariable *RegionBitmaps = nullptr;
220 uint32_t NumBitmapBytes = 0;
221
222 PerFunctionProfileData() = default;
223 };
225 // Key is virtual table variable, value is 'VTableProfData' in the form of
226 // GlobalVariable.
228 /// If runtime relocation is enabled, this maps functions to the load
229 /// instruction that produces the profile relocation bias.
230 DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;
231 std::vector<GlobalValue *> CompilerUsedVars;
232 std::vector<GlobalValue *> UsedVars;
233 std::vector<GlobalVariable *> ReferencedNames;
234 // The list of virtual table variables of which the VTableProfData is
235 // collected.
236 std::vector<GlobalVariable *> ReferencedVTables;
237 GlobalVariable *NamesVar = nullptr;
238 size_t NamesSize = 0;
239
240 // vector of counter load/store pairs to be register promoted.
241 std::vector<LoadStorePair> PromotionCandidates;
242
243 int64_t TotalCountersPromoted = 0;
244
245 /// Lower instrumentation intrinsics in the function. Returns true if there
246 /// any lowering.
248
249 /// Register-promote counter loads and stores in loops.
250 void promoteCounterLoadStores(Function *F);
251
252 /// Returns true if relocating counters at runtime is enabled.
253 bool isRuntimeCounterRelocationEnabled() const;
254
255 /// Returns true if profile counter update register promotion is enabled.
256 bool isCounterPromotionEnabled() const;
257
258 /// Count the number of instrumented value sites for the function.
259 void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
260
261 /// Replace instrprof.value.profile with a call to runtime library.
262 void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
263
264 /// Replace instrprof.cover with a store instruction to the coverage byte.
265 void lowerCover(InstrProfCoverInst *Inc);
266
267 /// Replace instrprof.timestamp with a call to
268 /// INSTR_PROF_PROFILE_SET_TIMESTAMP.
269 void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction);
270
271 /// Replace instrprof.increment with an increment of the appropriate value.
272 void lowerIncrement(InstrProfIncrementInst *Inc);
273
274 /// Force emitting of name vars for unused functions.
275 void lowerCoverageData(GlobalVariable *CoverageNamesVar);
276
277 /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction
278 /// using the index represented by the a temp value into a bitmap.
279 void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins);
280
281 /// Replace instrprof.mcdc.temp.update with a shift and or instruction using
282 /// the corresponding condition ID.
283 void lowerMCDCCondBitmapUpdate(InstrProfMCDCCondBitmapUpdate *Ins);
284
285 /// Compute the address of the counter value that this profiling instruction
286 /// acts on.
287 Value *getCounterAddress(InstrProfCntrInstBase *I);
288
289 /// Get the region counters for an increment, creating them if necessary.
290 ///
291 /// If the counter array doesn't yet exist, the profile data variables
292 /// referring to them will also be created.
293 GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc);
294
295 /// Create the region counters.
296 GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc,
299
300 /// Compute the address of the test vector bitmap that this profiling
301 /// instruction acts on.
302 Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I);
303
304 /// Get the region bitmaps for an increment, creating them if necessary.
305 ///
306 /// If the bitmap array doesn't yet exist, the profile data variables
307 /// referring to them will also be created.
308 GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc);
309
310 /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with
311 /// an MC/DC Decision region. The number of bytes required is indicated by
312 /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called
313 /// as part of setupProfileSection() and is conceptually very similar to
314 /// what is done for profile data counters in createRegionCounters().
315 GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
318
319 /// Set Comdat property of GV, if required.
320 void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName);
321
322 /// Setup the sections into which counters and bitmaps are allocated.
323 GlobalVariable *setupProfileSection(InstrProfInstBase *Inc,
324 InstrProfSectKind IPSK);
325
326 /// Create INSTR_PROF_DATA variable for counters and bitmaps.
327 void createDataVariable(InstrProfCntrInstBase *Inc);
328
329 /// Get the counters for virtual table values, creating them if necessary.
330 void getOrCreateVTableProfData(GlobalVariable *GV);
331
332 /// Emit the section with compressed function names.
333 void emitNameData();
334
335 /// Emit the section with compressed vtable names.
336 void emitVTableNames();
337
338 /// Emit value nodes section for value profiling.
339 void emitVNodes();
340
341 /// Emit runtime registration functions for each profile data variable.
342 void emitRegistration();
343
344 /// Emit the necessary plumbing to pull in the runtime initialization.
345 /// Returns true if a change was made.
346 bool emitRuntimeHook();
347
348 /// Add uses of our data variables and runtime hook.
349 void emitUses();
350
351 /// Create a static initializer for our data, on platforms that need it,
352 /// and for any profile output file that was specified.
353 void emitInitialization();
354};
355
356///
357/// A helper class to promote one counter RMW operation in the loop
358/// into register update.
359///
360/// RWM update for the counter will be sinked out of the loop after
361/// the transformation.
362///
363class PGOCounterPromoterHelper : public LoadAndStorePromoter {
364public:
365 PGOCounterPromoterHelper(
367 BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
368 ArrayRef<Instruction *> InsertPts,
370 LoopInfo &LI)
371 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
372 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
373 assert(isa<LoadInst>(L));
374 assert(isa<StoreInst>(S));
375 SSA.AddAvailableValue(PH, Init);
376 }
377
378 void doExtraRewritesBeforeFinalDeletion() override {
379 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
380 BasicBlock *ExitBlock = ExitBlocks[i];
381 Instruction *InsertPos = InsertPts[i];
382 // Get LiveIn value into the ExitBlock. If there are multiple
383 // predecessors, the value is defined by a PHI node in this
384 // block.
385 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
386 Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
387 Type *Ty = LiveInValue->getType();
388 IRBuilder<> Builder(InsertPos);
389 if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Addr)) {
390 // If isRuntimeCounterRelocationEnabled() is true then the address of
391 // the store instruction is computed with two instructions in
392 // InstrProfiling::getCounterAddress(). We need to copy those
393 // instructions to this block to compute Addr correctly.
394 // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
395 // %Addr = inttoptr i64 %BiasAdd to i64*
396 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
397 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
398 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
399 Addr = Builder.CreateIntToPtr(BiasInst,
400 PointerType::getUnqual(Ty->getContext()));
401 }
402 if (AtomicCounterUpdatePromoted)
403 // automic update currently can only be promoted across the current
404 // loop, not the whole loop nest.
405 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
406 MaybeAlign(),
407 AtomicOrdering::SequentiallyConsistent);
408 else {
409 LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
410 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
411 auto *NewStore = Builder.CreateStore(NewVal, Addr);
412
413 // Now update the parent loop's candidate list:
414 if (IterativeCounterPromotion) {
415 auto *TargetLoop = LI.getLoopFor(ExitBlock);
416 if (TargetLoop)
417 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
418 }
419 }
420 }
421 }
422
423private:
425 ArrayRef<BasicBlock *> ExitBlocks;
426 ArrayRef<Instruction *> InsertPts;
428 LoopInfo &LI;
429};
430
431/// A helper class to do register promotion for all profile counter
432/// updates in a loop.
433///
434class PGOCounterPromoter {
435public:
436 PGOCounterPromoter(
438 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
439 : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {
440
441 // Skip collection of ExitBlocks and InsertPts for loops that will not be
442 // able to have counters promoted.
443 SmallVector<BasicBlock *, 8> LoopExitBlocks;
445
446 L.getExitBlocks(LoopExitBlocks);
447 if (!isPromotionPossible(&L, LoopExitBlocks))
448 return;
449
450 for (BasicBlock *ExitBlock : LoopExitBlocks) {
451 if (BlockSet.insert(ExitBlock).second &&
452 llvm::none_of(predecessors(ExitBlock), [&](const BasicBlock *Pred) {
453 return llvm::isPresplitCoroSuspendExitEdge(*Pred, *ExitBlock);
454 })) {
455 ExitBlocks.push_back(ExitBlock);
456 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
457 }
458 }
459 }
460
461 bool run(int64_t *NumPromoted) {
462 // Skip 'infinite' loops:
463 if (ExitBlocks.size() == 0)
464 return false;
465
466 // Skip if any of the ExitBlocks contains a ret instruction.
467 // This is to prevent dumping of incomplete profile -- if the
468 // the loop is a long running loop and dump is called in the middle
469 // of the loop, the result profile is incomplete.
470 // FIXME: add other heuristics to detect long running loops.
471 if (SkipRetExitBlock) {
472 for (auto *BB : ExitBlocks)
473 if (isa<ReturnInst>(BB->getTerminator()))
474 return false;
475 }
476
477 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
478 if (MaxProm == 0)
479 return false;
480
481 unsigned Promoted = 0;
482 for (auto &Cand : LoopToCandidates[&L]) {
483
485 SSAUpdater SSA(&NewPHIs);
486 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
487
488 // If BFI is set, we will use it to guide the promotions.
489 if (BFI) {
490 auto *BB = Cand.first->getParent();
491 auto InstrCount = BFI->getBlockProfileCount(BB);
492 if (!InstrCount)
493 continue;
494 auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
495 // If the average loop trip count is not greater than 1.5, we skip
496 // promotion.
497 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
498 continue;
499 }
500
501 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
502 L.getLoopPreheader(), ExitBlocks,
503 InsertPts, LoopToCandidates, LI);
504 Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
505 Promoted++;
506 if (Promoted >= MaxProm)
507 break;
508
509 (*NumPromoted)++;
510 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
511 break;
512 }
513
514 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
515 << L.getLoopDepth() << ")\n");
516 return Promoted != 0;
517 }
518
519private:
520 bool allowSpeculativeCounterPromotion(Loop *LP) {
521 SmallVector<BasicBlock *, 8> ExitingBlocks;
522 L.getExitingBlocks(ExitingBlocks);
523 // Not considierered speculative.
524 if (ExitingBlocks.size() == 1)
525 return true;
526 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
527 return false;
528 return true;
529 }
530
531 // Check whether the loop satisfies the basic conditions needed to perform
532 // Counter Promotions.
533 bool
534 isPromotionPossible(Loop *LP,
535 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
536 // We can't insert into a catchswitch.
537 if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
538 return isa<CatchSwitchInst>(Exit->getTerminator());
539 }))
540 return false;
541
542 if (!LP->hasDedicatedExits())
543 return false;
544
545 BasicBlock *PH = LP->getLoopPreheader();
546 if (!PH)
547 return false;
548
549 return true;
550 }
551
552 // Returns the max number of Counter Promotions for LP.
553 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
554 SmallVector<BasicBlock *, 8> LoopExitBlocks;
555 LP->getExitBlocks(LoopExitBlocks);
556 if (!isPromotionPossible(LP, LoopExitBlocks))
557 return 0;
558
559 SmallVector<BasicBlock *, 8> ExitingBlocks;
560 LP->getExitingBlocks(ExitingBlocks);
561
562 // If BFI is set, we do more aggressive promotions based on BFI.
563 if (BFI)
564 return (unsigned)-1;
565
566 // Not considierered speculative.
567 if (ExitingBlocks.size() == 1)
568 return MaxNumOfPromotionsPerLoop;
569
570 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
571 return 0;
572
573 // Whether the target block is in a loop does not matter:
574 if (SpeculativeCounterPromotionToLoop)
575 return MaxNumOfPromotionsPerLoop;
576
577 // Now check the target block:
578 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
579 for (auto *TargetBlock : LoopExitBlocks) {
580 auto *TargetLoop = LI.getLoopFor(TargetBlock);
581 if (!TargetLoop)
582 continue;
583 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
584 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
585 MaxProm =
586 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
587 PendingCandsInTarget);
588 }
589 return MaxProm;
590 }
591
595 Loop &L;
596 LoopInfo &LI;
598};
599
600enum class ValueProfilingCallType {
601 // Individual values are tracked. Currently used for indiret call target
602 // profiling.
603 Default,
604
605 // MemOp: the memop size value profiling.
606 MemOp
607};
608
609} // end anonymous namespace
610
615 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
617 };
618 InstrLowerer Lowerer(M, Options, GetTLI, IsCS);
619 if (!Lowerer.lower())
620 return PreservedAnalyses::all();
621
623}
624
625bool InstrLowerer::lowerIntrinsics(Function *F) {
626 bool MadeChange = false;
627 PromotionCandidates.clear();
628 for (BasicBlock &BB : *F) {
629 for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
630 if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
631 lowerIncrement(IPIS);
632 MadeChange = true;
633 } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
634 lowerIncrement(IPI);
635 MadeChange = true;
636 } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(&Instr)) {
637 lowerTimestamp(IPC);
638 MadeChange = true;
639 } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
640 lowerCover(IPC);
641 MadeChange = true;
642 } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
643 lowerValueProfileInst(IPVP);
644 MadeChange = true;
645 } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(&Instr)) {
646 IPMP->eraseFromParent();
647 MadeChange = true;
648 } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(&Instr)) {
649 lowerMCDCTestVectorBitmapUpdate(IPBU);
650 MadeChange = true;
651 } else if (auto *IPTU = dyn_cast<InstrProfMCDCCondBitmapUpdate>(&Instr)) {
652 lowerMCDCCondBitmapUpdate(IPTU);
653 MadeChange = true;
654 }
655 }
656 }
657
658 if (!MadeChange)
659 return false;
660
661 promoteCounterLoadStores(F);
662 return true;
663}
664
665bool InstrLowerer::isRuntimeCounterRelocationEnabled() const {
666 // Mach-O don't support weak external references.
667 if (TT.isOSBinFormatMachO())
668 return false;
669
670 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
671 return RuntimeCounterRelocation;
672
673 // Fuchsia uses runtime counter relocation by default.
674 return TT.isOSFuchsia();
675}
676
677bool InstrLowerer::isCounterPromotionEnabled() const {
678 if (DoCounterPromotion.getNumOccurrences() > 0)
679 return DoCounterPromotion;
680
681 return Options.DoCounterPromotion;
682}
683
684void InstrLowerer::promoteCounterLoadStores(Function *F) {
685 if (!isCounterPromotionEnabled())
686 return;
687
688 DominatorTree DT(*F);
689 LoopInfo LI(DT);
690 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
691
692 std::unique_ptr<BlockFrequencyInfo> BFI;
693 if (Options.UseBFIInPromotion) {
694 std::unique_ptr<BranchProbabilityInfo> BPI;
695 BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
696 BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
697 }
698
699 for (const auto &LoadStore : PromotionCandidates) {
700 auto *CounterLoad = LoadStore.first;
701 auto *CounterStore = LoadStore.second;
702 BasicBlock *BB = CounterLoad->getParent();
703 Loop *ParentLoop = LI.getLoopFor(BB);
704 if (!ParentLoop)
705 continue;
706 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
707 }
708
710
711 // Do a post-order traversal of the loops so that counter updates can be
712 // iteratively hoisted outside the loop nest.
713 for (auto *Loop : llvm::reverse(Loops)) {
714 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
715 Promoter.run(&TotalCountersPromoted);
716 }
717}
718
720 // On Fuchsia, we only need runtime hook if any counters are present.
721 if (TT.isOSFuchsia())
722 return false;
723
724 return true;
725}
726
727/// Check if the module contains uses of any profiling intrinsics.
729 auto containsIntrinsic = [&](int ID) {
730 if (auto *F = M.getFunction(Intrinsic::getName(ID)))
731 return !F->use_empty();
732 return false;
733 };
734 return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
735 containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
736 containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
737 containsIntrinsic(llvm::Intrinsic::instrprof_timestamp) ||
738 containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
739}
740
741bool InstrLowerer::lower() {
742 bool MadeChange = false;
743 bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
744 if (NeedsRuntimeHook)
745 MadeChange = emitRuntimeHook();
746
747 bool ContainsProfiling = containsProfilingIntrinsics(M);
748 GlobalVariable *CoverageNamesVar =
749 M.getNamedGlobal(getCoverageUnusedNamesVarName());
750 // Improve compile time by avoiding linear scans when there is no work.
751 if (!ContainsProfiling && !CoverageNamesVar)
752 return MadeChange;
753
754 // We did not know how many value sites there would be inside
755 // the instrumented function. This is counting the number of instrumented
756 // target value sites to enter it as field in the profile data variable.
757 for (Function &F : M) {
758 InstrProfCntrInstBase *FirstProfInst = nullptr;
759 for (BasicBlock &BB : F) {
760 for (auto I = BB.begin(), E = BB.end(); I != E; I++) {
761 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
762 computeNumValueSiteCounts(Ind);
763 else {
764 if (FirstProfInst == nullptr &&
765 (isa<InstrProfIncrementInst>(I) || isa<InstrProfCoverInst>(I)))
766 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(I);
767 // If the MCDCBitmapParameters intrinsic seen, create the bitmaps.
768 if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(I))
769 static_cast<void>(getOrCreateRegionBitmaps(Params));
770 }
771 }
772 }
773
774 // Use a profile intrinsic to create the region counters and data variable.
775 // Also create the data variable based on the MCDCParams.
776 if (FirstProfInst != nullptr) {
777 static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
778 }
779 }
780
782 for (GlobalVariable &GV : M.globals())
783 // Global variables with type metadata are virtual table variables.
784 if (GV.hasMetadata(LLVMContext::MD_type))
785 getOrCreateVTableProfData(&GV);
786
787 for (Function &F : M)
788 MadeChange |= lowerIntrinsics(&F);
789
790 if (CoverageNamesVar) {
791 lowerCoverageData(CoverageNamesVar);
792 MadeChange = true;
793 }
794
795 if (!MadeChange)
796 return false;
797
798 emitVNodes();
799 emitNameData();
800 emitVTableNames();
801
802 // Emit runtime hook for the cases where the target does not unconditionally
803 // require pulling in profile runtime, and coverage is enabled on code that is
804 // not eliminated by the front-end, e.g. unused functions with internal
805 // linkage.
806 if (!NeedsRuntimeHook && ContainsProfiling)
807 emitRuntimeHook();
808
809 emitRegistration();
810 emitUses();
811 emitInitialization();
812 return true;
813}
814
816 Module &M, const TargetLibraryInfo &TLI,
817 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
818 LLVMContext &Ctx = M.getContext();
819 auto *ReturnTy = Type::getVoidTy(M.getContext());
820
821 AttributeList AL;
822 if (auto AK = TLI.getExtAttrForI32Param(false))
823 AL = AL.addParamAttribute(M.getContext(), 2, AK);
824
825 assert((CallType == ValueProfilingCallType::Default ||
826 CallType == ValueProfilingCallType::MemOp) &&
827 "Must be Default or MemOp");
828 Type *ParamTypes[] = {
829#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
831 };
832 auto *ValueProfilingCallTy =
833 FunctionType::get(ReturnTy, ArrayRef(ParamTypes), false);
834 StringRef FuncName = CallType == ValueProfilingCallType::Default
837 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
838}
839
840void InstrLowerer::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
841 GlobalVariable *Name = Ind->getName();
844 auto &PD = ProfileDataMap[Name];
845 PD.NumValueSites[ValueKind] =
846 std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));
847}
848
849void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
850 // TODO: Value profiling heavily depends on the data section which is omitted
851 // in lightweight mode. We need to move the value profile pointer to the
852 // Counter struct to get this working.
853 assert(
855 "Value profiling is not yet supported with lightweight instrumentation");
856 GlobalVariable *Name = Ind->getName();
857 auto It = ProfileDataMap.find(Name);
858 assert(It != ProfileDataMap.end() && It->second.DataVar &&
859 "value profiling detected in function with no counter incerement");
860
861 GlobalVariable *DataVar = It->second.DataVar;
864 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
865 Index += It->second.NumValueSites[Kind];
866
867 IRBuilder<> Builder(Ind);
868 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
869 llvm::InstrProfValueKind::IPVK_MemOPSize);
870 CallInst *Call = nullptr;
871 auto *TLI = &GetTLI(*Ind->getFunction());
872
873 // To support value profiling calls within Windows exception handlers, funclet
874 // information contained within operand bundles needs to be copied over to
875 // the library call. This is required for the IR to be processed by the
876 // WinEHPrepare pass.
878 Ind->getOperandBundlesAsDefs(OpBundles);
879 if (!IsMemOpSize) {
880 Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)};
881 Call = Builder.CreateCall(getOrInsertValueProfilingCall(M, *TLI), Args,
882 OpBundles);
883 } else {
884 Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)};
885 Call = Builder.CreateCall(
886 getOrInsertValueProfilingCall(M, *TLI, ValueProfilingCallType::MemOp),
887 Args, OpBundles);
888 }
889 if (auto AK = TLI->getExtAttrForI32Param(false))
890 Call->addParamAttr(2, AK);
891 Ind->replaceAllUsesWith(Call);
892 Ind->eraseFromParent();
893}
894
895Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
896 auto *Counters = getOrCreateRegionCounters(I);
897 IRBuilder<> Builder(I);
898
899 if (isa<InstrProfTimestampInst>(I))
900 Counters->setAlignment(Align(8));
901
902 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
903 Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());
904
905 if (!isRuntimeCounterRelocationEnabled())
906 return Addr;
907
908 Type *Int64Ty = Type::getInt64Ty(M.getContext());
909 Function *Fn = I->getParent()->getParent();
910 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
911 if (!BiasLI) {
912 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
913 auto *Bias = M.getGlobalVariable(getInstrProfCounterBiasVarName());
914 if (!Bias) {
915 // Compiler must define this variable when runtime counter relocation
916 // is being used. Runtime has a weak external reference that is used
917 // to check whether that's the case or not.
918 Bias = new GlobalVariable(
919 M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
921 Bias->setVisibility(GlobalVariable::HiddenVisibility);
922 // A definition that's weak (linkonce_odr) without being in a COMDAT
923 // section wouldn't lead to link errors, but it would lead to a dead
924 // data word from every TU but one. Putting it in COMDAT ensures there
925 // will be exactly one data slot in the link.
926 if (TT.supportsCOMDAT())
927 Bias->setComdat(M.getOrInsertComdat(Bias->getName()));
928 }
929 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
930 }
931 auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
932 return Builder.CreateIntToPtr(Add, Addr->getType());
933}
934
935Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
936 auto *Bitmaps = getOrCreateRegionBitmaps(I);
937 IRBuilder<> Builder(I);
938
939 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
940 Bitmaps->getValueType(), Bitmaps, 0, I->getBitmapIndex()->getZExtValue());
941
942 if (isRuntimeCounterRelocationEnabled()) {
943 LLVMContext &Ctx = M.getContext();
945 M.getName().data(),
946 Twine("Runtime counter relocation is presently not supported for MC/DC "
947 "bitmaps."),
948 DS_Warning));
949 }
950
951 return Addr;
952}
953
954void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
955 auto *Addr = getCounterAddress(CoverInstruction);
956 IRBuilder<> Builder(CoverInstruction);
957 // We store zero to represent that this block is covered.
958 Builder.CreateStore(Builder.getInt8(0), Addr);
959 CoverInstruction->eraseFromParent();
960}
961
962void InstrLowerer::lowerTimestamp(
963 InstrProfTimestampInst *TimestampInstruction) {
964 assert(TimestampInstruction->getIndex()->isZeroValue() &&
965 "timestamp probes are always the first probe for a function");
966 auto &Ctx = M.getContext();
967 auto *TimestampAddr = getCounterAddress(TimestampInstruction);
968 IRBuilder<> Builder(TimestampInstruction);
969 auto *CalleeTy =
970 FunctionType::get(Type::getVoidTy(Ctx), TimestampAddr->getType(), false);
971 auto Callee = M.getOrInsertFunction(
972 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy);
973 Builder.CreateCall(Callee, {TimestampAddr});
974 TimestampInstruction->eraseFromParent();
975}
976
977void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
978 auto *Addr = getCounterAddress(Inc);
979
980 IRBuilder<> Builder(Inc);
981 if (Options.Atomic || AtomicCounterUpdateAll ||
982 (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {
983 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
985 } else {
986 Value *IncStep = Inc->getStep();
987 Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
988 auto *Count = Builder.CreateAdd(Load, Inc->getStep());
989 auto *Store = Builder.CreateStore(Count, Addr);
990 if (isCounterPromotionEnabled())
991 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
992 }
993 Inc->eraseFromParent();
994}
995
996void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
997 ConstantArray *Names =
998 cast<ConstantArray>(CoverageNamesVar->getInitializer());
999 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
1000 Constant *NC = Names->getOperand(I);
1001 Value *V = NC->stripPointerCasts();
1002 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
1003 GlobalVariable *Name = cast<GlobalVariable>(V);
1004
1005 Name->setLinkage(GlobalValue::PrivateLinkage);
1006 ReferencedNames.push_back(Name);
1007 if (isa<ConstantExpr>(NC))
1008 NC->dropAllReferences();
1009 }
1010 CoverageNamesVar->eraseFromParent();
1011}
1012
1013void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
1015 IRBuilder<> Builder(Update);
1016 auto *Int8Ty = Type::getInt8Ty(M.getContext());
1017 auto *Int32Ty = Type::getInt32Ty(M.getContext());
1018 auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
1019 auto *BitmapAddr = getBitmapAddress(Update);
1020
1021 // Load Temp Val.
1022 // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
1023 auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp");
1024
1025 // Calculate byte offset using div8.
1026 // %1 = lshr i32 %mcdc.temp, 3
1027 auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3);
1028
1029 // Add byte offset to section base byte address.
1030 // %4 = getelementptr inbounds i8, ptr @__profbm_test, i32 %1
1031 auto *BitmapByteAddr =
1032 Builder.CreateInBoundsPtrAdd(BitmapAddr, BitmapByteOffset);
1033
1034 // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8)
1035 // %5 = and i32 %mcdc.temp, 7
1036 // %6 = trunc i32 %5 to i8
1037 auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty);
1038
1039 // Shift bit offset left to form a bitmap.
1040 // %7 = shl i8 1, %6
1041 auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);
1042
1043 // Load profile bitmap byte.
1044 // %mcdc.bits = load i8, ptr %4, align 1
1045 auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr, "mcdc.bits");
1046
1047 // Perform logical OR of profile bitmap byte and shifted bit offset.
1048 // %8 = or i8 %mcdc.bits, %7
1049 auto *Result = Builder.CreateOr(Bitmap, ShiftedVal);
1050
1051 // Store the updated profile bitmap byte.
1052 // store i8 %8, ptr %3, align 1
1053 Builder.CreateStore(Result, BitmapByteAddr);
1054 Update->eraseFromParent();
1055}
1056
1057void InstrLowerer::lowerMCDCCondBitmapUpdate(
1059 IRBuilder<> Builder(Update);
1060 auto *Int32Ty = Type::getInt32Ty(M.getContext());
1061 auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
1062
1063 // Load the MCDC temporary value from the stack.
1064 // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
1065 auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp");
1066
1067 // Zero-extend the evaluated condition boolean value (0 or 1) by 32bits.
1068 // %1 = zext i1 %tobool to i32
1069 auto *CondV_32 = Builder.CreateZExt(Update->getCondBool(), Int32Ty);
1070
1071 // Shift the boolean value left (by the condition's ID) to form a bitmap.
1072 // %2 = shl i32 %1, <Update->getCondID()>
1073 auto *ShiftedVal = Builder.CreateShl(CondV_32, Update->getCondID());
1074
1075 // Perform logical OR of the bitmap against the loaded MCDC temporary value.
1076 // %3 = or i32 %mcdc.temp, %2
1077 auto *Result = Builder.CreateOr(Temp, ShiftedVal);
1078
1079 // Store the updated temporary value back to the stack.
1080 // store i32 %3, ptr %mcdc.addr, align 4
1081 Builder.CreateStore(Result, MCDCCondBitmapAddr);
1082 Update->eraseFromParent();
1083}
1084
1085/// Get the name of a profiling variable for a particular function.
1086static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
1087 bool &Renamed) {
1088 StringRef NamePrefix = getInstrProfNameVarPrefix();
1089 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
1090 Function *F = Inc->getParent()->getParent();
1091 Module *M = F->getParent();
1092 if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
1094 Renamed = false;
1095 return (Prefix + Name).str();
1096 }
1097 Renamed = true;
1098 uint64_t FuncHash = Inc->getHash()->getZExtValue();
1099 SmallVector<char, 24> HashPostfix;
1100 if (Name.ends_with((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
1101 return (Prefix + Name).str();
1102 return (Prefix + Name + "." + Twine(FuncHash)).str();
1103}
1104
1106 // Only record function addresses if IR PGO is enabled or if clang value
1107 // profiling is enabled. Recording function addresses greatly increases object
1108 // file size, because it prevents the inliner from deleting functions that
1109 // have been inlined everywhere.
1110 if (!profDataReferencedByCode(*F->getParent()))
1111 return false;
1112
1113 // Check the linkage
1114 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
1115 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
1116 !HasAvailableExternallyLinkage)
1117 return true;
1118
1119 // A function marked 'alwaysinline' with available_externally linkage can't
1120 // have its address taken. Doing so would create an undefined external ref to
1121 // the function, which would fail to link.
1122 if (HasAvailableExternallyLinkage &&
1123 F->hasFnAttribute(Attribute::AlwaysInline))
1124 return false;
1125
1126 // Prohibit function address recording if the function is both internal and
1127 // COMDAT. This avoids the profile data variable referencing internal symbols
1128 // in COMDAT.
1129 if (F->hasLocalLinkage() && F->hasComdat())
1130 return false;
1131
1132 // Check uses of this function for other than direct calls or invokes to it.
1133 // Inline virtual functions have linkeOnceODR linkage. When a key method
1134 // exists, the vtable will only be emitted in the TU where the key method
1135 // is defined. In a TU where vtable is not available, the function won't
1136 // be 'addresstaken'. If its address is not recorded here, the profile data
1137 // with missing address may be picked by the linker leading to missing
1138 // indirect call target info.
1139 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
1140}
1141
1142static inline bool shouldUsePublicSymbol(Function *Fn) {
1143 // It isn't legal to make an alias of this function at all
1144 if (Fn->isDeclarationForLinker())
1145 return true;
1146
1147 // Symbols with local linkage can just use the symbol directly without
1148 // introducing relocations
1149 if (Fn->hasLocalLinkage())
1150 return true;
1151
1152 // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some
1153 // unfavorable interaction between the new alias and the alias renaming done
1154 // in LowerTypeTests under ThinLTO. For comdat functions that would normally
1155 // be deduplicated, but the renaming scheme ends up preventing renaming, since
1156 // it creates unique names for each alias, resulting in duplicated symbols. In
1157 // the future, we should update the CFI related passes to migrate these
1158 // aliases to the same module as the jump-table they refer to will be defined.
1159 if (Fn->hasMetadata(LLVMContext::MD_type))
1160 return true;
1161
1162 // For comdat functions, an alias would need the same linkage as the original
1163 // function and hidden visibility. There is no point in adding an alias with
1164 // identical linkage an visibility to avoid introducing symbolic relocations.
1165 if (Fn->hasComdat() &&
1167 return true;
1168
1169 // its OK to use an alias
1170 return false;
1171}
1172
1174 auto *Int8PtrTy = PointerType::getUnqual(Fn->getContext());
1175 // Store a nullptr in __llvm_profd, if we shouldn't use a real address
1176 if (!shouldRecordFunctionAddr(Fn))
1177 return ConstantPointerNull::get(Int8PtrTy);
1178
1179 // If we can't use an alias, we must use the public symbol, even though this
1180 // may require a symbolic relocation.
1181 if (shouldUsePublicSymbol(Fn))
1182 return Fn;
1183
1184 // When possible use a private alias to avoid symbolic relocations.
1186 Fn->getName() + ".local", Fn);
1187
1188 // When the instrumented function is a COMDAT function, we cannot use a
1189 // private alias. If we did, we would create reference to a local label in
1190 // this function's section. If this version of the function isn't selected by
1191 // the linker, then the metadata would introduce a reference to a discarded
1192 // section. So, for COMDAT functions, we need to adjust the linkage of the
1193 // alias. Using hidden visibility avoids a dynamic relocation and an entry in
1194 // the dynamic symbol table.
1195 //
1196 // Note that this handles COMDAT functions with visibility other than Hidden,
1197 // since that case is covered in shouldUsePublicSymbol()
1198 if (Fn->hasComdat()) {
1199 GA->setLinkage(Fn->getLinkage());
1201 }
1202
1203 // appendToCompilerUsed(*Fn->getParent(), {GA});
1204
1205 return GA;
1206}
1207
1209 // compiler-rt uses linker support to get data/counters/name start/end for
1210 // ELF, COFF, Mach-O and XCOFF.
1211 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
1212 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF())
1213 return false;
1214
1215 return true;
1216}
1217
1218void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO,
1219 StringRef CounterGroupName) {
1220 // Place lowered global variables in a comdat group if the associated function
1221 // or global variable is a COMDAT. This will make sure that only one copy of
1222 // global variable (e.g. function counters) of the COMDAT function will be
1223 // emitted after linking.
1224 bool NeedComdat = needsComdatForCounter(*GO, M);
1225 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
1226
1227 if (!UseComdat)
1228 return;
1229
1230 // Keep in mind that this pass may run before the inliner, so we need to
1231 // create a new comdat group (for counters, profiling data, etc). If we use
1232 // the comdat of the parent function, that will result in relocations against
1233 // discarded sections.
1234 //
1235 // If the data variable is referenced by code, non-counter variables (notably
1236 // profiling data) and counters have to be in different comdats for COFF
1237 // because the Visual C++ linker will report duplicate symbol errors if there
1238 // are multiple external symbols with the same name marked
1239 // IMAGE_COMDAT_SELECT_ASSOCIATIVE.
1240 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
1241 ? GV->getName()
1242 : CounterGroupName;
1243 Comdat *C = M.getOrInsertComdat(GroupName);
1244
1245 if (!NeedComdat) {
1246 // Object file format must be ELF since `UseComdat && !NeedComdat` is true.
1247 //
1248 // For ELF, when not using COMDAT, put counters, data and values into a
1249 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
1250 // allows -z start-stop-gc to discard the entire group when the function is
1251 // discarded.
1252 C->setSelectionKind(Comdat::NoDeduplicate);
1253 }
1254 GV->setComdat(C);
1255 // COFF doesn't allow the comdat group leader to have private linkage, so
1256 // upgrade private linkage to internal linkage to produce a symbol table
1257 // entry.
1258 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
1260}
1261
1263 if (!profDataReferencedByCode(*GV->getParent()))
1264 return false;
1265
1266 if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&
1268 return true;
1269
1270 // This avoids the profile data from referencing internal symbols in
1271 // COMDAT.
1272 if (GV->hasLocalLinkage() && GV->hasComdat())
1273 return false;
1274
1275 return true;
1276}
1277
1278// FIXME: Introduce an internal alias like what's done for functions to reduce
1279// the number of relocation entries.
1281 auto *Int8PtrTy = PointerType::getUnqual(GV->getContext());
1282
1283 // Store a nullptr in __profvt_ if a real address shouldn't be used.
1284 if (!shouldRecordVTableAddr(GV))
1285 return ConstantPointerNull::get(Int8PtrTy);
1286
1287 return ConstantExpr::getBitCast(GV, Int8PtrTy);
1288}
1289
1290void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) {
1292 "Value profiling is not supported with lightweight instrumentation");
1294 return;
1295
1296 // Skip llvm internal global variable or __prof variables.
1297 if (GV->getName().starts_with("llvm.") ||
1298 GV->getName().starts_with("__llvm") ||
1299 GV->getName().starts_with("__prof"))
1300 return;
1301
1302 // VTableProfData already created
1303 auto It = VTableDataMap.find(GV);
1304 if (It != VTableDataMap.end() && It->second)
1305 return;
1306
1309
1310 // This is to keep consistent with per-function profile data
1311 // for correctness.
1312 if (TT.isOSBinFormatXCOFF()) {
1314 Visibility = GlobalValue::DefaultVisibility;
1315 }
1316
1317 LLVMContext &Ctx = M.getContext();
1318 Type *DataTypes[] = {
1319#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
1321#undef INSTR_PROF_VTABLE_DATA
1322 };
1323
1324 auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));
1325
1326 // Used by INSTR_PROF_VTABLE_DATA MACRO
1327 Constant *VTableAddr = getVTableAddrForProfData(GV);
1328 const std::string PGOVTableName = getPGOName(*GV);
1329 // Record the length of the vtable. This is needed since vtable pointers
1330 // loaded from C++ objects might be from the middle of a vtable definition.
1331 uint32_t VTableSizeVal =
1332 M.getDataLayout().getTypeAllocSize(GV->getValueType());
1333
1334 Constant *DataVals[] = {
1335#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
1337#undef INSTR_PROF_VTABLE_DATA
1338 };
1339
1340 auto *Data =
1341 new GlobalVariable(M, DataTy, /*constant=*/false, Linkage,
1342 ConstantStruct::get(DataTy, DataVals),
1343 getInstrProfVTableVarPrefix() + PGOVTableName);
1344
1345 Data->setVisibility(Visibility);
1346 Data->setSection(getInstrProfSectionName(IPSK_vtab, TT.getObjectFormat()));
1347 Data->setAlignment(Align(8));
1348
1349 maybeSetComdat(Data, GV, Data->getName());
1350
1351 VTableDataMap[GV] = Data;
1352
1353 ReferencedVTables.push_back(GV);
1354
1355 // VTable <Hash, Addr> is used by runtime but not referenced by other
1356 // sections. Conservatively mark it linker retained.
1357 UsedVars.push_back(Data);
1358}
1359
1360GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc,
1361 InstrProfSectKind IPSK) {
1362 GlobalVariable *NamePtr = Inc->getName();
1363
1364 // Match the linkage and visibility of the name global.
1365 Function *Fn = Inc->getParent()->getParent();
1367 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1368
1369 // Use internal rather than private linkage so the counter variable shows up
1370 // in the symbol table when using debug info for correlation.
1371 if ((DebugInfoCorrelate ||
1373 TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage)
1375
1376 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1377 // symbols in the same csect won't be discarded. When there are duplicate weak
1378 // symbols, we can NOT guarantee that the relocations get resolved to the
1379 // intended weak symbol, so we can not ensure the correctness of the relative
1380 // CounterPtr, so we have to use private linkage for counter and data symbols.
1381 if (TT.isOSBinFormatXCOFF()) {
1383 Visibility = GlobalValue::DefaultVisibility;
1384 }
1385 // Move the name variable to the right section.
1386 bool Renamed;
1388 StringRef VarPrefix;
1389 std::string VarName;
1390 if (IPSK == IPSK_cnts) {
1391 VarPrefix = getInstrProfCountersVarPrefix();
1392 VarName = getVarName(Inc, VarPrefix, Renamed);
1393 InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Inc);
1394 Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);
1395 } else if (IPSK == IPSK_bitmap) {
1396 VarPrefix = getInstrProfBitmapVarPrefix();
1397 VarName = getVarName(Inc, VarPrefix, Renamed);
1398 InstrProfMCDCBitmapInstBase *BitmapUpdate =
1399 dyn_cast<InstrProfMCDCBitmapInstBase>(Inc);
1400 Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);
1401 } else {
1402 llvm_unreachable("Profile Section must be for Counters or Bitmaps");
1403 }
1404
1405 Ptr->setVisibility(Visibility);
1406 // Put the counters and bitmaps in their own sections so linkers can
1407 // remove unneeded sections.
1408 Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat()));
1409 Ptr->setLinkage(Linkage);
1410 maybeSetComdat(Ptr, Fn, VarName);
1411 return Ptr;
1412}
1413
1415InstrLowerer::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
1417 GlobalValue::LinkageTypes Linkage) {
1418 uint64_t NumBytes = Inc->getNumBitmapBytes()->getZExtValue();
1419 auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M.getContext()), NumBytes);
1420 auto GV = new GlobalVariable(M, BitmapTy, false, Linkage,
1421 Constant::getNullValue(BitmapTy), Name);
1422 GV->setAlignment(Align(1));
1423 return GV;
1424}
1425
1427InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
1428 GlobalVariable *NamePtr = Inc->getName();
1429 auto &PD = ProfileDataMap[NamePtr];
1430 if (PD.RegionBitmaps)
1431 return PD.RegionBitmaps;
1432
1433 // If RegionBitmaps doesn't already exist, create it by first setting up
1434 // the corresponding profile section.
1435 auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
1436 PD.RegionBitmaps = BitmapPtr;
1437 PD.NumBitmapBytes = Inc->getNumBitmapBytes()->getZExtValue();
1438 return PD.RegionBitmaps;
1439}
1440
1442InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
1443 GlobalValue::LinkageTypes Linkage) {
1444 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1445 auto &Ctx = M.getContext();
1446 GlobalVariable *GV;
1447 if (isa<InstrProfCoverInst>(Inc)) {
1448 auto *CounterTy = Type::getInt8Ty(Ctx);
1449 auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);
1450 // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
1451 std::vector<Constant *> InitialValues(NumCounters,
1452 Constant::getAllOnesValue(CounterTy));
1453 GV = new GlobalVariable(M, CounterArrTy, false, Linkage,
1454 ConstantArray::get(CounterArrTy, InitialValues),
1455 Name);
1456 GV->setAlignment(Align(1));
1457 } else {
1458 auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
1459 GV = new GlobalVariable(M, CounterTy, false, Linkage,
1460 Constant::getNullValue(CounterTy), Name);
1461 GV->setAlignment(Align(8));
1462 }
1463 return GV;
1464}
1465
1467InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
1468 GlobalVariable *NamePtr = Inc->getName();
1469 auto &PD = ProfileDataMap[NamePtr];
1470 if (PD.RegionCounters)
1471 return PD.RegionCounters;
1472
1473 // If RegionCounters doesn't already exist, create it by first setting up
1474 // the corresponding profile section.
1475 auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
1476 PD.RegionCounters = CounterPtr;
1477
1478 if (DebugInfoCorrelate ||
1480 LLVMContext &Ctx = M.getContext();
1481 Function *Fn = Inc->getParent()->getParent();
1482 if (auto *SP = Fn->getSubprogram()) {
1483 DIBuilder DB(M, true, SP->getUnit());
1484 Metadata *FunctionNameAnnotation[] = {
1487 };
1488 Metadata *CFGHashAnnotation[] = {
1491 };
1492 Metadata *NumCountersAnnotation[] = {
1495 };
1496 auto Annotations = DB.getOrCreateArray({
1497 MDNode::get(Ctx, FunctionNameAnnotation),
1498 MDNode::get(Ctx, CFGHashAnnotation),
1499 MDNode::get(Ctx, NumCountersAnnotation),
1500 });
1501 auto *DICounter = DB.createGlobalVariableExpression(
1502 SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
1503 /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),
1504 CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
1505 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1506 Annotations);
1507 CounterPtr->addDebugInfo(DICounter);
1508 DB.finalize();
1509 }
1510
1511 // Mark the counter variable as used so that it isn't optimized out.
1512 CompilerUsedVars.push_back(PD.RegionCounters);
1513 }
1514
1515 // Create the data variable (if it doesn't already exist).
1516 createDataVariable(Inc);
1517
1518 return PD.RegionCounters;
1519}
1520
1521void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
1522 // When debug information is correlated to profile data, a data variable
1523 // is not needed.
1525 return;
1526
1527 GlobalVariable *NamePtr = Inc->getName();
1528 auto &PD = ProfileDataMap[NamePtr];
1529
1530 // Return if data variable was already created.
1531 if (PD.DataVar)
1532 return;
1533
1534 LLVMContext &Ctx = M.getContext();
1535
1536 Function *Fn = Inc->getParent()->getParent();
1538 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1539
1540 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1541 // symbols in the same csect won't be discarded. When there are duplicate weak
1542 // symbols, we can NOT guarantee that the relocations get resolved to the
1543 // intended weak symbol, so we can not ensure the correctness of the relative
1544 // CounterPtr, so we have to use private linkage for counter and data symbols.
1545 if (TT.isOSBinFormatXCOFF()) {
1547 Visibility = GlobalValue::DefaultVisibility;
1548 }
1549
1550 bool NeedComdat = needsComdatForCounter(*Fn, M);
1551 bool Renamed;
1552
1553 // The Data Variable section is anchored to profile counters.
1554 std::string CntsVarName =
1556 std::string DataVarName =
1557 getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
1558
1559 auto *Int8PtrTy = PointerType::getUnqual(Ctx);
1560 // Allocate statically the array of pointers to value profile nodes for
1561 // the current function.
1562 Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
1563 uint64_t NS = 0;
1564 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1565 NS += PD.NumValueSites[Kind];
1566 if (NS > 0 && ValueProfileStaticAlloc &&
1568 ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
1569 auto *ValuesVar = new GlobalVariable(
1570 M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
1571 getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
1572 ValuesVar->setVisibility(Visibility);
1573 setGlobalVariableLargeSection(TT, *ValuesVar);
1574 ValuesVar->setSection(
1575 getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
1576 ValuesVar->setAlignment(Align(8));
1577 maybeSetComdat(ValuesVar, Fn, CntsVarName);
1578 ValuesPtrExpr = ValuesVar;
1579 }
1580
1581 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1582 auto *CounterPtr = PD.RegionCounters;
1583
1584 uint64_t NumBitmapBytes = PD.NumBitmapBytes;
1585
1586 // Create data variable.
1587 auto *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext());
1588 auto *Int16Ty = Type::getInt16Ty(Ctx);
1589 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
1590 Type *DataTypes[] = {
1591#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1593 };
1594 auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));
1595
1596 Constant *FunctionAddr = getFuncAddrForProfData(Fn);
1597
1598 Constant *Int16ArrayVals[IPVK_Last + 1];
1599 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1600 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
1601
1602 // If the data variable is not referenced by code (if we don't emit
1603 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
1604 // data variable live under linker GC, the data variable can be private. This
1605 // optimization applies to ELF.
1606 //
1607 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
1608 // to be false.
1609 //
1610 // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
1611 // that other copies must have the same CFG and cannot have value profiling.
1612 // If no hash suffix, other profd copies may be referenced by code.
1613 if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1614 (TT.isOSBinFormatELF() ||
1615 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
1617 Visibility = GlobalValue::DefaultVisibility;
1618 }
1619 auto *Data =
1620 new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName);
1621 Constant *RelativeCounterPtr;
1622 GlobalVariable *BitmapPtr = PD.RegionBitmaps;
1623 Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);
1624 InstrProfSectKind DataSectionKind;
1625 // With binary profile correlation, profile data is not loaded into memory.
1626 // profile data must reference profile counter with an absolute relocation.
1628 DataSectionKind = IPSK_covdata;
1629 RelativeCounterPtr = ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy);
1630 if (BitmapPtr != nullptr)
1631 RelativeBitmapPtr = ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy);
1632 } else {
1633 // Reference the counter variable with a label difference (link-time
1634 // constant).
1635 DataSectionKind = IPSK_data;
1636 RelativeCounterPtr =
1637 ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
1638 ConstantExpr::getPtrToInt(Data, IntPtrTy));
1639 if (BitmapPtr != nullptr)
1640 RelativeBitmapPtr =
1642 ConstantExpr::getPtrToInt(Data, IntPtrTy));
1643 }
1644
1645 Constant *DataVals[] = {
1646#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1648 };
1649 Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
1650
1651 Data->setVisibility(Visibility);
1652 Data->setSection(
1653 getInstrProfSectionName(DataSectionKind, TT.getObjectFormat()));
1654 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
1655 maybeSetComdat(Data, Fn, CntsVarName);
1656
1657 PD.DataVar = Data;
1658
1659 // Mark the data variable as used so that it isn't stripped out.
1660 CompilerUsedVars.push_back(Data);
1661 // Now that the linkage set by the FE has been passed to the data and counter
1662 // variables, reset Name variable's linkage and visibility to private so that
1663 // it can be removed later by the compiler.
1665 // Collect the referenced names to be used by emitNameData.
1666 ReferencedNames.push_back(NamePtr);
1667}
1668
1669void InstrLowerer::emitVNodes() {
1670 if (!ValueProfileStaticAlloc)
1671 return;
1672
1673 // For now only support this on platforms that do
1674 // not require runtime registration to discover
1675 // named section start/end.
1677 return;
1678
1679 size_t TotalNS = 0;
1680 for (auto &PD : ProfileDataMap) {
1681 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1682 TotalNS += PD.second.NumValueSites[Kind];
1683 }
1684
1685 if (!TotalNS)
1686 return;
1687
1688 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1689// Heuristic for small programs with very few total value sites.
1690// The default value of vp-counters-per-site is chosen based on
1691// the observation that large apps usually have a low percentage
1692// of value sites that actually have any profile data, and thus
1693// the average number of counters per site is low. For small
1694// apps with very few sites, this may not be true. Bump up the
1695// number of counters in this case.
1696#define INSTR_PROF_MIN_VAL_COUNTS 10
1697 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
1698 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
1699
1700 auto &Ctx = M.getContext();
1701 Type *VNodeTypes[] = {
1702#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1704 };
1705 auto *VNodeTy = StructType::get(Ctx, ArrayRef(VNodeTypes));
1706
1707 ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
1708 auto *VNodesVar = new GlobalVariable(
1709 M, VNodesTy, false, GlobalValue::PrivateLinkage,
1711 setGlobalVariableLargeSection(TT, *VNodesVar);
1712 VNodesVar->setSection(
1713 getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
1714 VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(VNodesTy));
1715 // VNodesVar is used by runtime but not referenced via relocation by other
1716 // sections. Conservatively make it linker retained.
1717 UsedVars.push_back(VNodesVar);
1718}
1719
1720void InstrLowerer::emitNameData() {
1721 std::string UncompressedData;
1722
1723 if (ReferencedNames.empty())
1724 return;
1725
1726 std::string CompressedNameStr;
1727 if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
1729 report_fatal_error(Twine(toString(std::move(E))), false);
1730 }
1731
1732 auto &Ctx = M.getContext();
1733 auto *NamesVal =
1734 ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false);
1735 NamesVar = new GlobalVariable(M, NamesVal->getType(), true,
1738 NamesSize = CompressedNameStr.size();
1739 setGlobalVariableLargeSection(TT, *NamesVar);
1740 NamesVar->setSection(
1742 ? getInstrProfSectionName(IPSK_covname, TT.getObjectFormat())
1743 : getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
1744 // On COFF, it's important to reduce the alignment down to 1 to prevent the
1745 // linker from inserting padding before the start of the names section or
1746 // between names entries.
1747 NamesVar->setAlignment(Align(1));
1748 // NamesVar is used by runtime but not referenced via relocation by other
1749 // sections. Conservatively make it linker retained.
1750 UsedVars.push_back(NamesVar);
1751
1752 for (auto *NamePtr : ReferencedNames)
1753 NamePtr->eraseFromParent();
1754}
1755
1756void InstrLowerer::emitVTableNames() {
1757 if (!EnableVTableValueProfiling || ReferencedVTables.empty())
1758 return;
1759
1760 // Collect the PGO names of referenced vtables and compress them.
1761 std::string CompressedVTableNames;
1762 if (Error E = collectVTableStrings(ReferencedVTables, CompressedVTableNames,
1764 report_fatal_error(Twine(toString(std::move(E))), false);
1765 }
1766
1767 auto &Ctx = M.getContext();
1768 auto *VTableNamesVal = ConstantDataArray::getString(
1769 Ctx, StringRef(CompressedVTableNames), false /* AddNull */);
1770 GlobalVariable *VTableNamesVar =
1771 new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */,
1772 GlobalValue::PrivateLinkage, VTableNamesVal,
1774 VTableNamesVar->setSection(
1775 getInstrProfSectionName(IPSK_vname, TT.getObjectFormat()));
1776 VTableNamesVar->setAlignment(Align(1));
1777 // Make VTableNames linker retained.
1778 UsedVars.push_back(VTableNamesVar);
1779}
1780
1781void InstrLowerer::emitRegistration() {
1783 return;
1784
1785 // Construct the function.
1786 auto *VoidTy = Type::getVoidTy(M.getContext());
1787 auto *VoidPtrTy = PointerType::getUnqual(M.getContext());
1788 auto *Int64Ty = Type::getInt64Ty(M.getContext());
1789 auto *RegisterFTy = FunctionType::get(VoidTy, false);
1790 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1792 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1793 if (Options.NoRedZone)
1794 RegisterF->addFnAttr(Attribute::NoRedZone);
1795
1796 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1797 auto *RuntimeRegisterF =
1800
1801 IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", RegisterF));
1802 for (Value *Data : CompilerUsedVars)
1803 if (!isa<Function>(Data))
1804 IRB.CreateCall(RuntimeRegisterF, Data);
1805 for (Value *Data : UsedVars)
1806 if (Data != NamesVar && !isa<Function>(Data))
1807 IRB.CreateCall(RuntimeRegisterF, Data);
1808
1809 if (NamesVar) {
1810 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1811 auto *NamesRegisterTy =
1812 FunctionType::get(VoidTy, ArrayRef(ParamTypes), false);
1813 auto *NamesRegisterF =
1816 IRB.CreateCall(NamesRegisterF, {NamesVar, IRB.getInt64(NamesSize)});
1817 }
1818
1819 IRB.CreateRetVoid();
1820}
1821
1822bool InstrLowerer::emitRuntimeHook() {
1823 // We expect the linker to be invoked with -u<hook_var> flag for Linux
1824 // in which case there is no need to emit the external variable.
1825 if (TT.isOSLinux() || TT.isOSAIX())
1826 return false;
1827
1828 // If the module's provided its own runtime, we don't need to do anything.
1829 if (M.getGlobalVariable(getInstrProfRuntimeHookVarName()))
1830 return false;
1831
1832 // Declare an external variable that will pull in the runtime initialization.
1833 auto *Int32Ty = Type::getInt32Ty(M.getContext());
1834 auto *Var =
1837 Var->setVisibility(GlobalValue::HiddenVisibility);
1838
1839 if (TT.isOSBinFormatELF() && !TT.isPS()) {
1840 // Mark the user variable as used so that it isn't stripped out.
1841 CompilerUsedVars.push_back(Var);
1842 } else {
1843 // Make a function that uses it.
1847 User->addFnAttr(Attribute::NoInline);
1848 if (Options.NoRedZone)
1849 User->addFnAttr(Attribute::NoRedZone);
1850 User->setVisibility(GlobalValue::HiddenVisibility);
1851 if (TT.supportsCOMDAT())
1852 User->setComdat(M.getOrInsertComdat(User->getName()));
1853
1854 IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", User));
1855 auto *Load = IRB.CreateLoad(Int32Ty, Var);
1856 IRB.CreateRet(Load);
1857
1858 // Mark the function as used so that it isn't stripped out.
1859 CompilerUsedVars.push_back(User);
1860 }
1861 return true;
1862}
1863
1864void InstrLowerer::emitUses() {
1865 // The metadata sections are parallel arrays. Optimizers (e.g.
1866 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
1867 // we conservatively retain all unconditionally in the compiler.
1868 //
1869 // On ELF and Mach-O, the linker can guarantee the associated sections will be
1870 // retained or discarded as a unit, so llvm.compiler.used is sufficient.
1871 // Similarly on COFF, if prof data is not referenced by code we use one comdat
1872 // and ensure this GC property as well. Otherwise, we have to conservatively
1873 // make all of the sections retained by the linker.
1874 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
1875 (TT.isOSBinFormatCOFF() && !DataReferencedByCode))
1876 appendToCompilerUsed(M, CompilerUsedVars);
1877 else
1878 appendToUsed(M, CompilerUsedVars);
1879
1880 // We do not add proper references from used metadata sections to NamesVar and
1881 // VNodesVar, so we have to be conservative and place them in llvm.used
1882 // regardless of the target,
1883 appendToUsed(M, UsedVars);
1884}
1885
1886void InstrLowerer::emitInitialization() {
1887 // Create ProfileFileName variable. Don't don't this for the
1888 // context-sensitive instrumentation lowering: This lowering is after
1889 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1890 // have already create the variable before LTO/ThinLTO linking.
1891 if (!IsCS)
1892 createProfileFileNameVar(M, Options.InstrProfileOutput);
1893 Function *RegisterF = M.getFunction(getInstrProfRegFuncsName());
1894 if (!RegisterF)
1895 return;
1896
1897 // Create the initialization function.
1898 auto *VoidTy = Type::getVoidTy(M.getContext());
1899 auto *F = Function::Create(FunctionType::get(VoidTy, false),
1902 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1903 F->addFnAttr(Attribute::NoInline);
1904 if (Options.NoRedZone)
1905 F->addFnAttr(Attribute::NoRedZone);
1906
1907 // Add the basic block and the necessary calls.
1908 IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", F));
1909 IRB.CreateCall(RegisterF, {});
1910 IRB.CreateRetVoid();
1911
1912 appendToGlobalCtors(M, F, 0);
1913}
This file contains the simple types necessary to represent the attributes associated with functions a...
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:693
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static unsigned InstrCount
static bool lowerIntrinsics(Module &M)
#define LLVM_DEBUG(X)
Definition: Debug.h:101
@ Default
Definition: DwarfDebug.cpp:87
uint64_t Addr
std::string Name
Hexagon Hardware Loops
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static bool shouldRecordVTableAddr(GlobalVariable *GV)
static bool shouldRecordFunctionAddr(Function *F)
static bool needsRuntimeHookUnconditionally(const Triple &TT)
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
#define INSTR_PROF_MIN_VAL_COUNTS
static Constant * getFuncAddrForProfData(Function *Fn)
static bool shouldUsePublicSymbol(Function *Fn)
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
static Constant * getVTableAddrForProfData(GlobalVariable *GV)
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
static LVOptions Options
Definition: LVOptions.cpp:25
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Memory SSA
Definition: MemorySSA.cpp:72
Module.h This file contains the declarations for the Module class.
IntegerType * Int32Ty
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
if(VerifyEach)
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:321
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:473
Annotations lets you mark points and ranges inside source code, for tests:
Definition: Annotations.h:53
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:647
@ Add
*p = old + v
Definition: Instructions.h:764
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
iterator end()
Definition: BasicBlock.h:443
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:430
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:409
const Instruction & front() const
Definition: BasicBlock.h:453
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:199
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:206
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
This class represents a function call, abstracting a target machine's calling convention.
@ NoDeduplicate
No deduplication is performed.
Definition: Comdat.h:39
ConstantArray - Constant Array Declarations.
Definition: Constants.h:423
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1291
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:528
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2881
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2542
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2112
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2140
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:154
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1775
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1356
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:417
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
bool isZeroValue() const
Return true if the value is negative zero or null value.
Definition: Constants.cpp:76
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:168
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:164
const BasicBlock & getEntryBlock() const
Definition: Function.h:787
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1830
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:356
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
Definition: Globals.cpp:530
bool hasMetadata() const
Return true if this value has any metadata attached to it.
Definition: Value.h:589
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
Definition: Globals.cpp:133
void setComdat(Comdat *C)
Definition: Globals.cpp:202
bool hasComdat() const
Definition: GlobalObject.h:128
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:263
bool hasLinkOnceLinkage() const
Definition: GlobalValue.h:514
VisibilityTypes getVisibility() const
Definition: GlobalValue.h:247
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:286
LinkageTypes getLinkage() const
Definition: GlobalValue.h:545
bool hasLocalLinkage() const
Definition: GlobalValue.h:527
bool hasPrivateLinkage() const
Definition: GlobalValue.h:526
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:536
bool isDeclarationForLinker() const
Definition: GlobalValue.h:617
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:65
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:66
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:67
bool hasAvailableExternallyLinkage() const
Definition: GlobalValue.h:511
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:50
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:59
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:58
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:51
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:54
Type * getValueType() const
Definition: GlobalValue.h:295
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:467
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2666
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:631
A base class for all instrprof counter intrinsics.
ConstantInt * getIndex() const
ConstantInt * getNumCounters() const
static const char * FunctionNameAttributeName
static const char * CFGHashAttributeName
static const char * NumCountersAttributeName
This represents the llvm.instrprof.cover intrinsic.
This represents the llvm.instrprof.increment intrinsic.
A base class for all instrprof intrinsics.
GlobalVariable * getName() const
ConstantInt * getHash() const
A base class for instrprof mcdc intrinsics that require global bitmap bytes.
ConstantInt * getNumBitmapBytes() const
This represents the llvm.instrprof.mcdc.condbitmap.update intrinsic.
This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic.
This represents the llvm.instrprof.timestamp intrinsic.
This represents the llvm.instrprof.value.profile intrinsic.
ConstantInt * getIndex() const
ConstantInt * getValueKind() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
const BasicBlock * getParent() const
Definition: Instruction.h:152
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:87
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Helper class for promoting a collection of loads and stores into SSA Form using the SSAUpdater.
Definition: SSAUpdater.h:151
virtual void doExtraRewritesBeforeFinalDeletion()
This hook is invoked after all the stores are found and inserted as available values.
Definition: SSAUpdater.h:176
An instruction for reading from memory.
Definition: Instructions.h:184
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:44
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1541
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:600
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:109
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:112
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:115
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition: SSAUpdater.h:40
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:564
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:258
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:373
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
Definition: User.h:169
unsigned getNumOperands() const
Definition: User.h:191
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1074
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
ValueKind
Value kinds.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:1027
@ PD
PD - Prefix code for packed double precision vector floating point operations performed in the SSE re...
Definition: X86BaseInfo.h:735
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:718
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:92
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:163
StringRef getInstrProfBitmapVarPrefix()
Return the name prefix of profile bitmap variables.
Definition: InstrProf.h:104
cl::opt< bool > DoInstrProfNameCompression
cl::opt< InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate("profile-correlate", cl::desc("Use debug info or binary file to correlate profiles."), cl::init(InstrProfCorrelator::NONE), cl::values(clEnumValN(InstrProfCorrelator::NONE, "", "No profile correlation"), clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", "Use debug info to correlate"), clEnumValN(InstrProfCorrelator::BINARY, "binary", "Use binary to correlate")))
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:656
StringRef getInstrProfVTableNamesVarName()
Definition: InstrProf.h:116
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:98
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:129
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:231
cl::opt< bool > DebugInfoCorrelate
bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1408
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
Definition: InstrProf.cpp:390
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:158
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:107
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
StringRef getInstrProfCounterBiasVarName()
Definition: InstrProf.h:173
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:419
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition: InstrProf.h:169
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:138
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
Definition: InstrProf.cpp:703
InstrProfSectKind
Definition: InstrProf.h:60
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:101
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1736
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:696
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition: InstrProf.h:87
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:150
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
@ Add
Sum of integers.
Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
Definition: InstrProf.cpp:713
void setGlobalVariableLargeSection(const Triple &TargetTriple, GlobalVariable &GV)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1456
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1479
@ DS_Warning
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:74
bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src, const BasicBlock &Dest)
auto predecessors(const MachineBasicBlock *BB)
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:81
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:144
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition: InstrProf.h:114
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1434
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:110
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
StringRef getInstrProfVTableVarPrefix()
Return the name prefix of variables containing virtual table profile data.
Definition: InstrProf.h:95
#define NC
Definition: regutils.h:42
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Options for the frontend instrumentation based profiling pass.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117