LLVM 20.0.0git
InstrProfiling.cpp
Go to the documentation of this file.
1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by an instrumentor.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/Twine.h"
25#include "llvm/IR/Attributes.h"
26#include "llvm/IR/BasicBlock.h"
27#include "llvm/IR/CFG.h"
28#include "llvm/IR/Constant.h"
29#include "llvm/IR/Constants.h"
30#include "llvm/IR/DIBuilder.h"
33#include "llvm/IR/Dominators.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/GlobalValue.h"
37#include "llvm/IR/IRBuilder.h"
38#include "llvm/IR/Instruction.h"
41#include "llvm/IR/MDBuilder.h"
42#include "llvm/IR/Module.h"
43#include "llvm/IR/Type.h"
45#include "llvm/Pass.h"
50#include "llvm/Support/Error.h"
58#include <algorithm>
59#include <cassert>
60#include <cstdint>
61#include <string>
62
63using namespace llvm;
64
65#define DEBUG_TYPE "instrprof"
66
67namespace llvm {
68// Command line option to enable vtable value profiling. Defined in
69// ProfileData/InstrProf.cpp: -enable-vtable-value-profiling=
71// TODO: Remove -debug-info-correlate in next LLVM release, in favor of
72// -profile-correlate=debug-info.
74 "debug-info-correlate",
75 cl::desc("Use debug info to correlate profiles. (Deprecated, use "
76 "-profile-correlate=debug-info)"),
77 cl::init(false));
78
80 "profile-correlate",
81 cl::desc("Use debug info or binary file to correlate profiles."),
84 "No profile correlation"),
86 "Use debug info to correlate"),
88 "Use binary to correlate")));
89} // namespace llvm
90
91namespace {
92
93cl::opt<bool> DoHashBasedCounterSplit(
94 "hash-based-counter-split",
95 cl::desc("Rename counter variable of a comdat function based on cfg hash"),
96 cl::init(true));
97
99 RuntimeCounterRelocation("runtime-counter-relocation",
100 cl::desc("Enable relocating counters at runtime."),
101 cl::init(false));
102
103cl::opt<bool> ValueProfileStaticAlloc(
104 "vp-static-alloc",
105 cl::desc("Do static counter allocation for value profiler"),
106 cl::init(true));
107
108cl::opt<double> NumCountersPerValueSite(
109 "vp-counters-per-site",
110 cl::desc("The average number of profile counters allocated "
111 "per value profiling site."),
112 // This is set to a very small value because in real programs, only
113 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
114 // For those sites with non-zero profile, the average number of targets
115 // is usually smaller than 2.
116 cl::init(1.0));
117
118cl::opt<bool> AtomicCounterUpdateAll(
119 "instrprof-atomic-counter-update-all",
120 cl::desc("Make all profile counter updates atomic (for testing only)"),
121 cl::init(false));
122
123cl::opt<bool> AtomicCounterUpdatePromoted(
124 "atomic-counter-update-promoted",
125 cl::desc("Do counter update using atomic fetch add "
126 " for promoted counters only"),
127 cl::init(false));
128
129cl::opt<bool> AtomicFirstCounter(
130 "atomic-first-counter",
131 cl::desc("Use atomic fetch add for first counter in a function (usually "
132 "the entry counter)"),
133 cl::init(false));
134
135// If the option is not specified, the default behavior about whether
136// counter promotion is done depends on how instrumentaiton lowering
137// pipeline is setup, i.e., the default value of true of this option
138// does not mean the promotion will be done by default. Explicitly
139// setting this option can override the default behavior.
140cl::opt<bool> DoCounterPromotion("do-counter-promotion",
141 cl::desc("Do counter register promotion"),
142 cl::init(false));
143cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
144 "max-counter-promotions-per-loop", cl::init(20),
145 cl::desc("Max number counter promotions per loop to avoid"
146 " increasing register pressure too much"));
147
148// A debug option
150 MaxNumOfPromotions("max-counter-promotions", cl::init(-1),
151 cl::desc("Max number of allowed counter promotions"));
152
153cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
154 "speculative-counter-promotion-max-exiting", cl::init(3),
155 cl::desc("The max number of exiting blocks of a loop to allow "
156 " speculative counter promotion"));
157
158cl::opt<bool> SpeculativeCounterPromotionToLoop(
159 "speculative-counter-promotion-to-loop",
160 cl::desc("When the option is false, if the target block is in a loop, "
161 "the promotion will be disallowed unless the promoted counter "
162 " update can be further/iteratively promoted into an acyclic "
163 " region."));
164
165cl::opt<bool> IterativeCounterPromotion(
166 "iterative-counter-promotion", cl::init(true),
167 cl::desc("Allow counter promotion across the whole loop nest."));
168
169cl::opt<bool> SkipRetExitBlock(
170 "skip-ret-exit-block", cl::init(true),
171 cl::desc("Suppress counter promotion if exit blocks contain ret."));
172
173static cl::opt<bool> SampledInstr("sampled-instrumentation", cl::ZeroOrMore,
174 cl::init(false),
175 cl::desc("Do PGO instrumentation sampling"));
176
177static cl::opt<unsigned> SampledInstrPeriod(
178 "sampled-instr-period",
179 cl::desc("Set the profile instrumentation sample period. For each sample "
180 "period, a fixed number of consecutive samples will be recorded. "
181 "The number is controlled by 'sampled-instr-burst-duration' flag. "
182 "The default sample period of 65535 is optimized for generating "
183 "efficient code that leverages unsigned integer wrapping in "
184 "overflow."),
185 cl::init(65535));
186
187static cl::opt<unsigned> SampledInstrBurstDuration(
188 "sampled-instr-burst-duration",
189 cl::desc("Set the profile instrumentation burst duration, which can range "
190 "from 0 to one less than the value of 'sampled-instr-period'. "
191 "This number of samples will be recorded for each "
192 "'sampled-instr-period' count update. Setting to 1 enables "
193 "simple sampling, in which case it is recommended to set "
194 "'sampled-instr-period' to a prime number."),
195 cl::init(200));
196
197using LoadStorePair = std::pair<Instruction *, Instruction *>;
198
199static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag) {
200 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
201 if (!MD)
202 return 0;
203
204 // If the flag is a ConstantAsMetadata, it should be an integer representable
205 // in 64-bits.
206 return cast<ConstantInt>(MD->getValue())->getZExtValue();
207}
208
209static bool enablesValueProfiling(const Module &M) {
210 return isIRPGOFlagSet(&M) ||
211 getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;
212}
213
214// Conservatively returns true if value profiling is enabled.
215static bool profDataReferencedByCode(const Module &M) {
216 return enablesValueProfiling(M);
217}
218
219class InstrLowerer final {
220public:
221 InstrLowerer(Module &M, const InstrProfOptions &Options,
222 std::function<const TargetLibraryInfo &(Function &F)> GetTLI,
223 bool IsCS)
224 : M(M), Options(Options), TT(Triple(M.getTargetTriple())), IsCS(IsCS),
225 GetTLI(GetTLI), DataReferencedByCode(profDataReferencedByCode(M)) {}
226
227 bool lower();
228
229private:
230 Module &M;
232 const Triple TT;
233 // Is this lowering for the context-sensitive instrumentation.
234 const bool IsCS;
235
236 std::function<const TargetLibraryInfo &(Function &F)> GetTLI;
237
238 const bool DataReferencedByCode;
239
240 struct PerFunctionProfileData {
241 uint32_t NumValueSites[IPVK_Last + 1] = {};
242 GlobalVariable *RegionCounters = nullptr;
243 GlobalVariable *DataVar = nullptr;
244 GlobalVariable *RegionBitmaps = nullptr;
245 uint32_t NumBitmapBytes = 0;
246
247 PerFunctionProfileData() = default;
248 };
250 // Key is virtual table variable, value is 'VTableProfData' in the form of
251 // GlobalVariable.
253 /// If runtime relocation is enabled, this maps functions to the load
254 /// instruction that produces the profile relocation bias.
255 DenseMap<const Function *, LoadInst *> FunctionToProfileBiasMap;
256 std::vector<GlobalValue *> CompilerUsedVars;
257 std::vector<GlobalValue *> UsedVars;
258 std::vector<GlobalVariable *> ReferencedNames;
259 // The list of virtual table variables of which the VTableProfData is
260 // collected.
261 std::vector<GlobalVariable *> ReferencedVTables;
262 GlobalVariable *NamesVar = nullptr;
263 size_t NamesSize = 0;
264
265 /// The instance of [[alwaysinline]] rmw_or(ptr, i8).
266 /// This is name-insensitive.
267 Function *RMWOrFunc = nullptr;
268
269 // vector of counter load/store pairs to be register promoted.
270 std::vector<LoadStorePair> PromotionCandidates;
271
272 int64_t TotalCountersPromoted = 0;
273
274 /// Lower instrumentation intrinsics in the function. Returns true if there
275 /// any lowering.
277
278 /// Register-promote counter loads and stores in loops.
279 void promoteCounterLoadStores(Function *F);
280
281 /// Returns true if relocating counters at runtime is enabled.
282 bool isRuntimeCounterRelocationEnabled() const;
283
284 /// Returns true if profile counter update register promotion is enabled.
285 bool isCounterPromotionEnabled() const;
286
287 /// Return true if profile sampling is enabled.
288 bool isSamplingEnabled() const;
289
290 /// Count the number of instrumented value sites for the function.
291 void computeNumValueSiteCounts(InstrProfValueProfileInst *Ins);
292
293 /// Replace instrprof.value.profile with a call to runtime library.
294 void lowerValueProfileInst(InstrProfValueProfileInst *Ins);
295
296 /// Replace instrprof.cover with a store instruction to the coverage byte.
297 void lowerCover(InstrProfCoverInst *Inc);
298
299 /// Replace instrprof.timestamp with a call to
300 /// INSTR_PROF_PROFILE_SET_TIMESTAMP.
301 void lowerTimestamp(InstrProfTimestampInst *TimestampInstruction);
302
303 /// Replace instrprof.increment with an increment of the appropriate value.
304 void lowerIncrement(InstrProfIncrementInst *Inc);
305
306 /// Force emitting of name vars for unused functions.
307 void lowerCoverageData(GlobalVariable *CoverageNamesVar);
308
309 /// Replace instrprof.mcdc.tvbitmask.update with a shift and or instruction
310 /// using the index represented by the a temp value into a bitmap.
311 void lowerMCDCTestVectorBitmapUpdate(InstrProfMCDCTVBitmapUpdate *Ins);
312
313 /// Get the Bias value for data to access mmap-ed area.
314 /// Create it if it hasn't been seen.
315 GlobalVariable *getOrCreateBiasVar(StringRef VarName);
316
317 /// Compute the address of the counter value that this profiling instruction
318 /// acts on.
319 Value *getCounterAddress(InstrProfCntrInstBase *I);
320
321 /// Lower the incremental instructions under profile sampling predicates.
322 void doSampling(Instruction *I);
323
324 /// Get the region counters for an increment, creating them if necessary.
325 ///
326 /// If the counter array doesn't yet exist, the profile data variables
327 /// referring to them will also be created.
328 GlobalVariable *getOrCreateRegionCounters(InstrProfCntrInstBase *Inc);
329
330 /// Create the region counters.
331 GlobalVariable *createRegionCounters(InstrProfCntrInstBase *Inc,
334
335 /// Create [[alwaysinline]] rmw_or(ptr, i8).
336 /// This doesn't update `RMWOrFunc`.
337 Function *createRMWOrFunc();
338
339 /// Get the call to `rmw_or`.
340 /// Create the instance if it is unknown.
341 CallInst *getRMWOrCall(Value *Addr, Value *Val);
342
343 /// Compute the address of the test vector bitmap that this profiling
344 /// instruction acts on.
345 Value *getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I);
346
347 /// Get the region bitmaps for an increment, creating them if necessary.
348 ///
349 /// If the bitmap array doesn't yet exist, the profile data variables
350 /// referring to them will also be created.
351 GlobalVariable *getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc);
352
353 /// Create the MC/DC bitmap as a byte-aligned array of bytes associated with
354 /// an MC/DC Decision region. The number of bytes required is indicated by
355 /// the intrinsic used (type InstrProfMCDCBitmapInstBase). This is called
356 /// as part of setupProfileSection() and is conceptually very similar to
357 /// what is done for profile data counters in createRegionCounters().
358 GlobalVariable *createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
361
362 /// Set Comdat property of GV, if required.
363 void maybeSetComdat(GlobalVariable *GV, GlobalObject *GO, StringRef VarName);
364
365 /// Setup the sections into which counters and bitmaps are allocated.
366 GlobalVariable *setupProfileSection(InstrProfInstBase *Inc,
367 InstrProfSectKind IPSK);
368
369 /// Create INSTR_PROF_DATA variable for counters and bitmaps.
370 void createDataVariable(InstrProfCntrInstBase *Inc);
371
372 /// Get the counters for virtual table values, creating them if necessary.
373 void getOrCreateVTableProfData(GlobalVariable *GV);
374
375 /// Emit the section with compressed function names.
376 void emitNameData();
377
378 /// Emit the section with compressed vtable names.
379 void emitVTableNames();
380
381 /// Emit value nodes section for value profiling.
382 void emitVNodes();
383
384 /// Emit runtime registration functions for each profile data variable.
385 void emitRegistration();
386
387 /// Emit the necessary plumbing to pull in the runtime initialization.
388 /// Returns true if a change was made.
389 bool emitRuntimeHook();
390
391 /// Add uses of our data variables and runtime hook.
392 void emitUses();
393
394 /// Create a static initializer for our data, on platforms that need it,
395 /// and for any profile output file that was specified.
396 void emitInitialization();
397};
398
399///
400/// A helper class to promote one counter RMW operation in the loop
401/// into register update.
402///
403/// RWM update for the counter will be sinked out of the loop after
404/// the transformation.
405///
406class PGOCounterPromoterHelper : public LoadAndStorePromoter {
407public:
408 PGOCounterPromoterHelper(
410 BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
411 ArrayRef<Instruction *> InsertPts,
413 LoopInfo &LI)
414 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
415 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
416 assert(isa<LoadInst>(L));
417 assert(isa<StoreInst>(S));
418 SSA.AddAvailableValue(PH, Init);
419 }
420
421 void doExtraRewritesBeforeFinalDeletion() override {
422 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
423 BasicBlock *ExitBlock = ExitBlocks[i];
424 Instruction *InsertPos = InsertPts[i];
425 // Get LiveIn value into the ExitBlock. If there are multiple
426 // predecessors, the value is defined by a PHI node in this
427 // block.
428 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
429 Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
430 Type *Ty = LiveInValue->getType();
431 IRBuilder<> Builder(InsertPos);
432 if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Addr)) {
433 // If isRuntimeCounterRelocationEnabled() is true then the address of
434 // the store instruction is computed with two instructions in
435 // InstrProfiling::getCounterAddress(). We need to copy those
436 // instructions to this block to compute Addr correctly.
437 // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
438 // %Addr = inttoptr i64 %BiasAdd to i64*
439 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
440 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
441 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
442 Addr = Builder.CreateIntToPtr(BiasInst,
443 PointerType::getUnqual(Ty->getContext()));
444 }
445 if (AtomicCounterUpdatePromoted)
446 // automic update currently can only be promoted across the current
447 // loop, not the whole loop nest.
448 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
449 MaybeAlign(),
450 AtomicOrdering::SequentiallyConsistent);
451 else {
452 LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
453 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
454 auto *NewStore = Builder.CreateStore(NewVal, Addr);
455
456 // Now update the parent loop's candidate list:
457 if (IterativeCounterPromotion) {
458 auto *TargetLoop = LI.getLoopFor(ExitBlock);
459 if (TargetLoop)
460 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
461 }
462 }
463 }
464 }
465
466private:
468 ArrayRef<BasicBlock *> ExitBlocks;
469 ArrayRef<Instruction *> InsertPts;
471 LoopInfo &LI;
472};
473
474/// A helper class to do register promotion for all profile counter
475/// updates in a loop.
476///
477class PGOCounterPromoter {
478public:
479 PGOCounterPromoter(
481 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
482 : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {
483
484 // Skip collection of ExitBlocks and InsertPts for loops that will not be
485 // able to have counters promoted.
486 SmallVector<BasicBlock *, 8> LoopExitBlocks;
488
489 L.getExitBlocks(LoopExitBlocks);
490 if (!isPromotionPossible(&L, LoopExitBlocks))
491 return;
492
493 for (BasicBlock *ExitBlock : LoopExitBlocks) {
494 if (BlockSet.insert(ExitBlock).second &&
495 llvm::none_of(predecessors(ExitBlock), [&](const BasicBlock *Pred) {
496 return llvm::isPresplitCoroSuspendExitEdge(*Pred, *ExitBlock);
497 })) {
498 ExitBlocks.push_back(ExitBlock);
499 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
500 }
501 }
502 }
503
504 bool run(int64_t *NumPromoted) {
505 // Skip 'infinite' loops:
506 if (ExitBlocks.size() == 0)
507 return false;
508
509 // Skip if any of the ExitBlocks contains a ret instruction.
510 // This is to prevent dumping of incomplete profile -- if the
511 // the loop is a long running loop and dump is called in the middle
512 // of the loop, the result profile is incomplete.
513 // FIXME: add other heuristics to detect long running loops.
514 if (SkipRetExitBlock) {
515 for (auto *BB : ExitBlocks)
516 if (isa<ReturnInst>(BB->getTerminator()))
517 return false;
518 }
519
520 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
521 if (MaxProm == 0)
522 return false;
523
524 unsigned Promoted = 0;
525 for (auto &Cand : LoopToCandidates[&L]) {
526
528 SSAUpdater SSA(&NewPHIs);
529 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
530
531 // If BFI is set, we will use it to guide the promotions.
532 if (BFI) {
533 auto *BB = Cand.first->getParent();
534 auto InstrCount = BFI->getBlockProfileCount(BB);
535 if (!InstrCount)
536 continue;
537 auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
538 // If the average loop trip count is not greater than 1.5, we skip
539 // promotion.
540 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
541 continue;
542 }
543
544 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
545 L.getLoopPreheader(), ExitBlocks,
546 InsertPts, LoopToCandidates, LI);
547 Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
548 Promoted++;
549 if (Promoted >= MaxProm)
550 break;
551
552 (*NumPromoted)++;
553 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
554 break;
555 }
556
557 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
558 << L.getLoopDepth() << ")\n");
559 return Promoted != 0;
560 }
561
562private:
563 bool allowSpeculativeCounterPromotion(Loop *LP) {
564 SmallVector<BasicBlock *, 8> ExitingBlocks;
565 L.getExitingBlocks(ExitingBlocks);
566 // Not considierered speculative.
567 if (ExitingBlocks.size() == 1)
568 return true;
569 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
570 return false;
571 return true;
572 }
573
574 // Check whether the loop satisfies the basic conditions needed to perform
575 // Counter Promotions.
576 bool
577 isPromotionPossible(Loop *LP,
578 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
579 // We can't insert into a catchswitch.
580 if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
581 return isa<CatchSwitchInst>(Exit->getTerminator());
582 }))
583 return false;
584
585 if (!LP->hasDedicatedExits())
586 return false;
587
588 BasicBlock *PH = LP->getLoopPreheader();
589 if (!PH)
590 return false;
591
592 return true;
593 }
594
595 // Returns the max number of Counter Promotions for LP.
596 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
597 SmallVector<BasicBlock *, 8> LoopExitBlocks;
598 LP->getExitBlocks(LoopExitBlocks);
599 if (!isPromotionPossible(LP, LoopExitBlocks))
600 return 0;
601
602 SmallVector<BasicBlock *, 8> ExitingBlocks;
603 LP->getExitingBlocks(ExitingBlocks);
604
605 // If BFI is set, we do more aggressive promotions based on BFI.
606 if (BFI)
607 return (unsigned)-1;
608
609 // Not considierered speculative.
610 if (ExitingBlocks.size() == 1)
611 return MaxNumOfPromotionsPerLoop;
612
613 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
614 return 0;
615
616 // Whether the target block is in a loop does not matter:
617 if (SpeculativeCounterPromotionToLoop)
618 return MaxNumOfPromotionsPerLoop;
619
620 // Now check the target block:
621 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
622 for (auto *TargetBlock : LoopExitBlocks) {
623 auto *TargetLoop = LI.getLoopFor(TargetBlock);
624 if (!TargetLoop)
625 continue;
626 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
627 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
628 MaxProm =
629 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
630 PendingCandsInTarget);
631 }
632 return MaxProm;
633 }
634
638 Loop &L;
639 LoopInfo &LI;
641};
642
643enum class ValueProfilingCallType {
644 // Individual values are tracked. Currently used for indiret call target
645 // profiling.
646 Default,
647
648 // MemOp: the memop size value profiling.
649 MemOp
650};
651
652} // end anonymous namespace
653
658 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
660 };
661 InstrLowerer Lowerer(M, Options, GetTLI, IsCS);
662 if (!Lowerer.lower())
663 return PreservedAnalyses::all();
664
666}
667
668//
669// Perform instrumentation sampling.
670//
671// There are 3 favors of sampling:
672// (1) Full burst sampling: We transform:
673// Increment_Instruction;
674// to:
675// if (__llvm_profile_sampling__ < SampledInstrBurstDuration) {
676// Increment_Instruction;
677// }
678// __llvm_profile_sampling__ += 1;
679// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
680// __llvm_profile_sampling__ = 0;
681// }
682//
683// "__llvm_profile_sampling__" is a thread-local global shared by all PGO
684// counters (value-instrumentation and edge instrumentation).
685//
686// (2) Fast burst sampling:
687// "__llvm_profile_sampling__" variable is an unsigned type, meaning it will
688// wrap around to zero when overflows. In this case, the second check is
689// unnecessary, so we won't generate check2 when the SampledInstrPeriod is
690// set to 65535 (64K - 1). The code after:
691// if (__llvm_profile_sampling__ < SampledInstrBurstDuration) {
692// Increment_Instruction;
693// }
694// __llvm_profile_sampling__ += 1;
695//
696// (3) Simple sampling:
697// When SampledInstrBurstDuration sets to 1, we do a simple sampling:
698// __llvm_profile_sampling__ += 1;
699// if (__llvm_profile_sampling__ >= SampledInstrPeriod) {
700// __llvm_profile_sampling__ = 0;
701// Increment_Instruction;
702// }
703//
704// Note that, the code snippet after the transformation can still be counter
705// promoted. However, with sampling enabled, counter updates are expected to
706// be infrequent, making the benefits of counter promotion negligible.
707// Moreover, counter promotion can potentially cause issues in server
708// applications, particularly when the counters are dumped without a clean
709// exit. To mitigate this risk, counter promotion is disabled by default when
710// sampling is enabled. This behavior can be overridden using the internal
711// option.
712void InstrLowerer::doSampling(Instruction *I) {
713 if (!isSamplingEnabled())
714 return;
715
716 unsigned SampledBurstDuration = SampledInstrBurstDuration.getValue();
717 unsigned SampledPeriod = SampledInstrPeriod.getValue();
718 if (SampledBurstDuration >= SampledPeriod) {
720 "SampledPeriod needs to be greater than SampledBurstDuration");
721 }
722 bool UseShort = (SampledPeriod <= USHRT_MAX);
723 bool IsSimpleSampling = (SampledBurstDuration == 1);
724 // If (SampledBurstDuration == 1 && SampledPeriod == 65535), generate
725 // the simple sampling style code.
726 bool IsFastSampling = (!IsSimpleSampling && SampledPeriod == 65535);
727
728 auto GetConstant = [UseShort](IRBuilder<> &Builder, uint32_t C) {
729 if (UseShort)
730 return Builder.getInt16(C);
731 else
732 return Builder.getInt32(C);
733 };
734
735 IntegerType *SamplingVarTy;
736 if (UseShort)
737 SamplingVarTy = Type::getInt16Ty(M.getContext());
738 else
739 SamplingVarTy = Type::getInt32Ty(M.getContext());
740 auto *SamplingVar =
741 M.getGlobalVariable(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
742 assert(SamplingVar && "SamplingVar not set properly");
743
744 // Create the condition for checking the burst duration.
745 Instruction *SamplingVarIncr;
746 Value *NewSamplingVarVal;
747 MDBuilder MDB(I->getContext());
748 MDNode *BranchWeight;
749 IRBuilder<> CondBuilder(I);
750 auto *LoadSamplingVar = CondBuilder.CreateLoad(SamplingVarTy, SamplingVar);
751 if (IsSimpleSampling) {
752 // For the simple sampling, just create the load and increments.
753 IRBuilder<> IncBuilder(I);
754 NewSamplingVarVal =
755 IncBuilder.CreateAdd(LoadSamplingVar, GetConstant(IncBuilder, 1));
756 SamplingVarIncr = IncBuilder.CreateStore(NewSamplingVarVal, SamplingVar);
757 } else {
758 // For the bust-sampling, create the conditonal update.
759 auto *DurationCond = CondBuilder.CreateICmpULE(
760 LoadSamplingVar, GetConstant(CondBuilder, SampledBurstDuration));
761 BranchWeight = MDB.createBranchWeights(
762 SampledBurstDuration, SampledPeriod + 1 - SampledBurstDuration);
764 DurationCond, I, /* Unreachable */ false, BranchWeight);
765 IRBuilder<> IncBuilder(I);
766 NewSamplingVarVal =
767 IncBuilder.CreateAdd(LoadSamplingVar, GetConstant(IncBuilder, 1));
768 SamplingVarIncr = IncBuilder.CreateStore(NewSamplingVarVal, SamplingVar);
769 I->moveBefore(ThenTerm);
770 }
771
772 if (IsFastSampling)
773 return;
774
775 // Create the condtion for checking the period.
776 Instruction *ThenTerm, *ElseTerm;
777 IRBuilder<> PeriodCondBuilder(SamplingVarIncr);
778 auto *PeriodCond = PeriodCondBuilder.CreateICmpUGE(
779 NewSamplingVarVal, GetConstant(PeriodCondBuilder, SampledPeriod));
780 BranchWeight = MDB.createBranchWeights(1, SampledPeriod);
781 SplitBlockAndInsertIfThenElse(PeriodCond, SamplingVarIncr, &ThenTerm,
782 &ElseTerm, BranchWeight);
783
784 // For the simple sampling, the counter update happens in sampling var reset.
785 if (IsSimpleSampling)
786 I->moveBefore(ThenTerm);
787
788 IRBuilder<> ResetBuilder(ThenTerm);
789 ResetBuilder.CreateStore(GetConstant(ResetBuilder, 0), SamplingVar);
790 SamplingVarIncr->moveBefore(ElseTerm);
791}
792
793bool InstrLowerer::lowerIntrinsics(Function *F) {
794 bool MadeChange = false;
795 PromotionCandidates.clear();
797
798 // To ensure compatibility with sampling, we save the intrinsics into
799 // a buffer to prevent potential breakage of the iterator (as the
800 // intrinsics will be moved to a different BB).
801 for (BasicBlock &BB : *F) {
802 for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
803 if (auto *IP = dyn_cast<InstrProfInstBase>(&Instr))
804 InstrProfInsts.push_back(IP);
805 }
806 }
807
808 for (auto *Instr : InstrProfInsts) {
809 doSampling(Instr);
810 if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(Instr)) {
811 lowerIncrement(IPIS);
812 MadeChange = true;
813 } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(Instr)) {
814 lowerIncrement(IPI);
815 MadeChange = true;
816 } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(Instr)) {
817 lowerTimestamp(IPC);
818 MadeChange = true;
819 } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(Instr)) {
820 lowerCover(IPC);
821 MadeChange = true;
822 } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(Instr)) {
823 lowerValueProfileInst(IPVP);
824 MadeChange = true;
825 } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(Instr)) {
826 IPMP->eraseFromParent();
827 MadeChange = true;
828 } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(Instr)) {
829 lowerMCDCTestVectorBitmapUpdate(IPBU);
830 MadeChange = true;
831 }
832 }
833
834 if (!MadeChange)
835 return false;
836
837 promoteCounterLoadStores(F);
838 return true;
839}
840
841bool InstrLowerer::isRuntimeCounterRelocationEnabled() const {
842 // Mach-O don't support weak external references.
843 if (TT.isOSBinFormatMachO())
844 return false;
845
846 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
847 return RuntimeCounterRelocation;
848
849 // Fuchsia uses runtime counter relocation by default.
850 return TT.isOSFuchsia();
851}
852
853bool InstrLowerer::isSamplingEnabled() const {
854 if (SampledInstr.getNumOccurrences() > 0)
855 return SampledInstr;
856 return Options.Sampling;
857}
858
859bool InstrLowerer::isCounterPromotionEnabled() const {
860 if (DoCounterPromotion.getNumOccurrences() > 0)
861 return DoCounterPromotion;
862
863 return Options.DoCounterPromotion;
864}
865
866void InstrLowerer::promoteCounterLoadStores(Function *F) {
867 if (!isCounterPromotionEnabled())
868 return;
869
870 DominatorTree DT(*F);
871 LoopInfo LI(DT);
872 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
873
874 std::unique_ptr<BlockFrequencyInfo> BFI;
875 if (Options.UseBFIInPromotion) {
876 std::unique_ptr<BranchProbabilityInfo> BPI;
877 BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
878 BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
879 }
880
881 for (const auto &LoadStore : PromotionCandidates) {
882 auto *CounterLoad = LoadStore.first;
883 auto *CounterStore = LoadStore.second;
884 BasicBlock *BB = CounterLoad->getParent();
885 Loop *ParentLoop = LI.getLoopFor(BB);
886 if (!ParentLoop)
887 continue;
888 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
889 }
890
892
893 // Do a post-order traversal of the loops so that counter updates can be
894 // iteratively hoisted outside the loop nest.
895 for (auto *Loop : llvm::reverse(Loops)) {
896 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
897 Promoter.run(&TotalCountersPromoted);
898 }
899}
900
902 // On Fuchsia, we only need runtime hook if any counters are present.
903 if (TT.isOSFuchsia())
904 return false;
905
906 return true;
907}
908
909/// Check if the module contains uses of any profiling intrinsics.
911 auto containsIntrinsic = [&](int ID) {
912 if (auto *F = M.getFunction(Intrinsic::getName(ID)))
913 return !F->use_empty();
914 return false;
915 };
916 return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
917 containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
918 containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
919 containsIntrinsic(llvm::Intrinsic::instrprof_timestamp) ||
920 containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
921}
922
923bool InstrLowerer::lower() {
924 bool MadeChange = false;
925 bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
926 if (NeedsRuntimeHook)
927 MadeChange = emitRuntimeHook();
928
929 if (!IsCS && isSamplingEnabled())
931
932 bool ContainsProfiling = containsProfilingIntrinsics(M);
933 GlobalVariable *CoverageNamesVar =
934 M.getNamedGlobal(getCoverageUnusedNamesVarName());
935 // Improve compile time by avoiding linear scans when there is no work.
936 if (!ContainsProfiling && !CoverageNamesVar)
937 return MadeChange;
938
939 // We did not know how many value sites there would be inside
940 // the instrumented function. This is counting the number of instrumented
941 // target value sites to enter it as field in the profile data variable.
942 for (Function &F : M) {
943 InstrProfCntrInstBase *FirstProfInst = nullptr;
944 for (BasicBlock &BB : F) {
945 for (auto I = BB.begin(), E = BB.end(); I != E; I++) {
946 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
947 computeNumValueSiteCounts(Ind);
948 else {
949 if (FirstProfInst == nullptr &&
950 (isa<InstrProfIncrementInst>(I) || isa<InstrProfCoverInst>(I)))
951 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(I);
952 // If the MCDCBitmapParameters intrinsic seen, create the bitmaps.
953 if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(I))
954 static_cast<void>(getOrCreateRegionBitmaps(Params));
955 }
956 }
957 }
958
959 // Use a profile intrinsic to create the region counters and data variable.
960 // Also create the data variable based on the MCDCParams.
961 if (FirstProfInst != nullptr) {
962 static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
963 }
964 }
965
967 for (GlobalVariable &GV : M.globals())
968 // Global variables with type metadata are virtual table variables.
969 if (GV.hasMetadata(LLVMContext::MD_type))
970 getOrCreateVTableProfData(&GV);
971
972 for (Function &F : M)
973 MadeChange |= lowerIntrinsics(&F);
974
975 if (CoverageNamesVar) {
976 lowerCoverageData(CoverageNamesVar);
977 MadeChange = true;
978 }
979
980 if (!MadeChange)
981 return false;
982
983 emitVNodes();
984 emitNameData();
985 emitVTableNames();
986
987 // Emit runtime hook for the cases where the target does not unconditionally
988 // require pulling in profile runtime, and coverage is enabled on code that is
989 // not eliminated by the front-end, e.g. unused functions with internal
990 // linkage.
991 if (!NeedsRuntimeHook && ContainsProfiling)
992 emitRuntimeHook();
993
994 emitRegistration();
995 emitUses();
996 emitInitialization();
997 return true;
998}
999
1001 Module &M, const TargetLibraryInfo &TLI,
1002 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
1003 LLVMContext &Ctx = M.getContext();
1004 auto *ReturnTy = Type::getVoidTy(M.getContext());
1005
1006 AttributeList AL;
1007 if (auto AK = TLI.getExtAttrForI32Param(false))
1008 AL = AL.addParamAttribute(M.getContext(), 2, AK);
1009
1010 assert((CallType == ValueProfilingCallType::Default ||
1011 CallType == ValueProfilingCallType::MemOp) &&
1012 "Must be Default or MemOp");
1013 Type *ParamTypes[] = {
1014#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
1016 };
1017 auto *ValueProfilingCallTy =
1018 FunctionType::get(ReturnTy, ArrayRef(ParamTypes), false);
1019 StringRef FuncName = CallType == ValueProfilingCallType::Default
1022 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
1023}
1024
1025void InstrLowerer::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
1026 GlobalVariable *Name = Ind->getName();
1028 uint64_t Index = Ind->getIndex()->getZExtValue();
1029 auto &PD = ProfileDataMap[Name];
1030 PD.NumValueSites[ValueKind] =
1031 std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));
1032}
1033
1034void InstrLowerer::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
1035 // TODO: Value profiling heavily depends on the data section which is omitted
1036 // in lightweight mode. We need to move the value profile pointer to the
1037 // Counter struct to get this working.
1038 assert(
1040 "Value profiling is not yet supported with lightweight instrumentation");
1041 GlobalVariable *Name = Ind->getName();
1042 auto It = ProfileDataMap.find(Name);
1043 assert(It != ProfileDataMap.end() && It->second.DataVar &&
1044 "value profiling detected in function with no counter incerement");
1045
1046 GlobalVariable *DataVar = It->second.DataVar;
1048 uint64_t Index = Ind->getIndex()->getZExtValue();
1049 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
1050 Index += It->second.NumValueSites[Kind];
1051
1052 IRBuilder<> Builder(Ind);
1053 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
1054 llvm::InstrProfValueKind::IPVK_MemOPSize);
1055 CallInst *Call = nullptr;
1056 auto *TLI = &GetTLI(*Ind->getFunction());
1057
1058 // To support value profiling calls within Windows exception handlers, funclet
1059 // information contained within operand bundles needs to be copied over to
1060 // the library call. This is required for the IR to be processed by the
1061 // WinEHPrepare pass.
1063 Ind->getOperandBundlesAsDefs(OpBundles);
1064 if (!IsMemOpSize) {
1065 Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)};
1066 Call = Builder.CreateCall(getOrInsertValueProfilingCall(M, *TLI), Args,
1067 OpBundles);
1068 } else {
1069 Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)};
1070 Call = Builder.CreateCall(
1071 getOrInsertValueProfilingCall(M, *TLI, ValueProfilingCallType::MemOp),
1072 Args, OpBundles);
1073 }
1074 if (auto AK = TLI->getExtAttrForI32Param(false))
1075 Call->addParamAttr(2, AK);
1076 Ind->replaceAllUsesWith(Call);
1077 Ind->eraseFromParent();
1078}
1079
1080GlobalVariable *InstrLowerer::getOrCreateBiasVar(StringRef VarName) {
1081 GlobalVariable *Bias = M.getGlobalVariable(VarName);
1082 if (Bias)
1083 return Bias;
1084
1085 Type *Int64Ty = Type::getInt64Ty(M.getContext());
1086
1087 // Compiler must define this variable when runtime counter relocation
1088 // is being used. Runtime has a weak external reference that is used
1089 // to check whether that's the case or not.
1090 Bias = new GlobalVariable(M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
1091 Constant::getNullValue(Int64Ty), VarName);
1093 // A definition that's weak (linkonce_odr) without being in a COMDAT
1094 // section wouldn't lead to link errors, but it would lead to a dead
1095 // data word from every TU but one. Putting it in COMDAT ensures there
1096 // will be exactly one data slot in the link.
1097 if (TT.supportsCOMDAT())
1098 Bias->setComdat(M.getOrInsertComdat(VarName));
1099
1100 return Bias;
1101}
1102
1103Value *InstrLowerer::getCounterAddress(InstrProfCntrInstBase *I) {
1104 auto *Counters = getOrCreateRegionCounters(I);
1105 IRBuilder<> Builder(I);
1106
1107 if (isa<InstrProfTimestampInst>(I))
1108 Counters->setAlignment(Align(8));
1109
1110 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
1111 Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());
1112
1113 if (!isRuntimeCounterRelocationEnabled())
1114 return Addr;
1115
1116 Type *Int64Ty = Type::getInt64Ty(M.getContext());
1117 Function *Fn = I->getParent()->getParent();
1118 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
1119 if (!BiasLI) {
1120 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1121 auto *Bias = getOrCreateBiasVar(getInstrProfCounterBiasVarName());
1122 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias, "profc_bias");
1123 // Bias doesn't change after startup.
1124 BiasLI->setMetadata(LLVMContext::MD_invariant_load,
1125 MDNode::get(M.getContext(), std::nullopt));
1126 }
1127 auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
1128 return Builder.CreateIntToPtr(Add, Addr->getType());
1129}
1130
1131/// Create `void [[alwaysinline]] rmw_or(uint8_t *ArgAddr, uint8_t ArgVal)`
1132/// "Basic" sequence is `*ArgAddr |= ArgVal`
1133Function *InstrLowerer::createRMWOrFunc() {
1134 auto &Ctx = M.getContext();
1135 auto *Int8Ty = Type::getInt8Ty(Ctx);
1138 {PointerType::getUnqual(Ctx), Int8Ty}, false),
1140 Fn->addFnAttr(Attribute::AlwaysInline);
1141 auto *ArgAddr = Fn->getArg(0);
1142 auto *ArgVal = Fn->getArg(1);
1143 IRBuilder<> Builder(BasicBlock::Create(Ctx, "", Fn));
1144
1145 // Load profile bitmap byte.
1146 // %mcdc.bits = load i8, ptr %4, align 1
1147 auto *Bitmap = Builder.CreateLoad(Int8Ty, ArgAddr, "mcdc.bits");
1148
1149 if (Options.Atomic || AtomicCounterUpdateAll) {
1150 // If ((Bitmap & Val) != Val), then execute atomic (Bitmap |= Val).
1151 // Note, just-loaded Bitmap might not be up-to-date. Use it just for
1152 // early testing.
1153 auto *Masked = Builder.CreateAnd(Bitmap, ArgVal);
1154 auto *ShouldStore = Builder.CreateICmpNE(Masked, ArgVal);
1155 auto *ThenTerm = BasicBlock::Create(Ctx, "", Fn);
1156 auto *ElseTerm = BasicBlock::Create(Ctx, "", Fn);
1157 // Assume updating will be rare.
1158 auto *Unlikely = MDBuilder(Ctx).createUnlikelyBranchWeights();
1159 Builder.CreateCondBr(ShouldStore, ThenTerm, ElseTerm, Unlikely);
1160
1161 IRBuilder<> ThenBuilder(ThenTerm);
1162 ThenBuilder.CreateAtomicRMW(AtomicRMWInst::Or, ArgAddr, ArgVal,
1164 ThenBuilder.CreateRetVoid();
1165
1166 IRBuilder<> ElseBuilder(ElseTerm);
1167 ElseBuilder.CreateRetVoid();
1168
1169 return Fn;
1170 }
1171
1172 // Perform logical OR of profile bitmap byte and shifted bit offset.
1173 // %8 = or i8 %mcdc.bits, %7
1174 auto *Result = Builder.CreateOr(Bitmap, ArgVal);
1175
1176 // Store the updated profile bitmap byte.
1177 // store i8 %8, ptr %3, align 1
1178 Builder.CreateStore(Result, ArgAddr);
1179
1180 // Terminator
1181 Builder.CreateRetVoid();
1182
1183 return Fn;
1184}
1185
1186CallInst *InstrLowerer::getRMWOrCall(Value *Addr, Value *Val) {
1187 if (!RMWOrFunc)
1188 RMWOrFunc = createRMWOrFunc();
1189
1190 return CallInst::Create(RMWOrFunc, {Addr, Val});
1191}
1192
1193Value *InstrLowerer::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
1194 auto *Bitmaps = getOrCreateRegionBitmaps(I);
1195 if (!isRuntimeCounterRelocationEnabled())
1196 return Bitmaps;
1197
1198 // Put BiasLI onto the entry block.
1199 Type *Int64Ty = Type::getInt64Ty(M.getContext());
1200 Function *Fn = I->getFunction();
1201 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
1202 auto *Bias = getOrCreateBiasVar(getInstrProfBitmapBiasVarName());
1203 auto *BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias, "profbm_bias");
1204 // Assume BiasLI invariant (in the function at least)
1205 BiasLI->setMetadata(LLVMContext::MD_invariant_load,
1206 MDNode::get(M.getContext(), std::nullopt));
1207
1208 // Add Bias to Bitmaps and put it before the intrinsic.
1209 IRBuilder<> Builder(I);
1210 return Builder.CreatePtrAdd(Bitmaps, BiasLI, "profbm_addr");
1211}
1212
1213void InstrLowerer::lowerCover(InstrProfCoverInst *CoverInstruction) {
1214 auto *Addr = getCounterAddress(CoverInstruction);
1215 IRBuilder<> Builder(CoverInstruction);
1216 // We store zero to represent that this block is covered.
1217 Builder.CreateStore(Builder.getInt8(0), Addr);
1218 CoverInstruction->eraseFromParent();
1219}
1220
1221void InstrLowerer::lowerTimestamp(
1222 InstrProfTimestampInst *TimestampInstruction) {
1223 assert(TimestampInstruction->getIndex()->isZeroValue() &&
1224 "timestamp probes are always the first probe for a function");
1225 auto &Ctx = M.getContext();
1226 auto *TimestampAddr = getCounterAddress(TimestampInstruction);
1227 IRBuilder<> Builder(TimestampInstruction);
1228 auto *CalleeTy =
1229 FunctionType::get(Type::getVoidTy(Ctx), TimestampAddr->getType(), false);
1230 auto Callee = M.getOrInsertFunction(
1231 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy);
1232 Builder.CreateCall(Callee, {TimestampAddr});
1233 TimestampInstruction->eraseFromParent();
1234}
1235
1236void InstrLowerer::lowerIncrement(InstrProfIncrementInst *Inc) {
1237 auto *Addr = getCounterAddress(Inc);
1238
1239 IRBuilder<> Builder(Inc);
1240 if (Options.Atomic || AtomicCounterUpdateAll ||
1241 (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {
1244 } else {
1245 Value *IncStep = Inc->getStep();
1246 Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
1247 auto *Count = Builder.CreateAdd(Load, Inc->getStep());
1248 auto *Store = Builder.CreateStore(Count, Addr);
1249 if (isCounterPromotionEnabled())
1250 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
1251 }
1252 Inc->eraseFromParent();
1253}
1254
1255void InstrLowerer::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
1256 ConstantArray *Names =
1257 cast<ConstantArray>(CoverageNamesVar->getInitializer());
1258 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
1259 Constant *NC = Names->getOperand(I);
1260 Value *V = NC->stripPointerCasts();
1261 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
1262 GlobalVariable *Name = cast<GlobalVariable>(V);
1263
1264 Name->setLinkage(GlobalValue::PrivateLinkage);
1265 ReferencedNames.push_back(Name);
1266 if (isa<ConstantExpr>(NC))
1267 NC->dropAllReferences();
1268 }
1269 CoverageNamesVar->eraseFromParent();
1270}
1271
1272void InstrLowerer::lowerMCDCTestVectorBitmapUpdate(
1274 IRBuilder<> Builder(Update);
1275 auto *Int8Ty = Type::getInt8Ty(M.getContext());
1276 auto *Int32Ty = Type::getInt32Ty(M.getContext());
1277 auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
1278 auto *BitmapAddr = getBitmapAddress(Update);
1279
1280 // Load Temp Val + BitmapIdx.
1281 // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
1282 auto *Temp = Builder.CreateAdd(
1283 Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp"),
1284 Update->getBitmapIndex());
1285
1286 // Calculate byte offset using div8.
1287 // %1 = lshr i32 %mcdc.temp, 3
1288 auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3);
1289
1290 // Add byte offset to section base byte address.
1291 // %4 = getelementptr inbounds i8, ptr @__profbm_test, i32 %1
1292 auto *BitmapByteAddr =
1293 Builder.CreateInBoundsPtrAdd(BitmapAddr, BitmapByteOffset);
1294
1295 // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8)
1296 // %5 = and i32 %mcdc.temp, 7
1297 // %6 = trunc i32 %5 to i8
1298 auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty);
1299
1300 // Shift bit offset left to form a bitmap.
1301 // %7 = shl i8 1, %6
1302 auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);
1303
1304 Builder.Insert(getRMWOrCall(BitmapByteAddr, ShiftedVal));
1305 Update->eraseFromParent();
1306}
1307
1308/// Get the name of a profiling variable for a particular function.
1309static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
1310 bool &Renamed) {
1311 StringRef NamePrefix = getInstrProfNameVarPrefix();
1312 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
1313 Function *F = Inc->getParent()->getParent();
1314 Module *M = F->getParent();
1315 if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
1317 Renamed = false;
1318 return (Prefix + Name).str();
1319 }
1320 Renamed = true;
1321 uint64_t FuncHash = Inc->getHash()->getZExtValue();
1322 SmallVector<char, 24> HashPostfix;
1323 if (Name.ends_with((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
1324 return (Prefix + Name).str();
1325 return (Prefix + Name + "." + Twine(FuncHash)).str();
1326}
1327
1329 // Only record function addresses if IR PGO is enabled or if clang value
1330 // profiling is enabled. Recording function addresses greatly increases object
1331 // file size, because it prevents the inliner from deleting functions that
1332 // have been inlined everywhere.
1333 if (!profDataReferencedByCode(*F->getParent()))
1334 return false;
1335
1336 // Check the linkage
1337 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
1338 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
1339 !HasAvailableExternallyLinkage)
1340 return true;
1341
1342 // A function marked 'alwaysinline' with available_externally linkage can't
1343 // have its address taken. Doing so would create an undefined external ref to
1344 // the function, which would fail to link.
1345 if (HasAvailableExternallyLinkage &&
1346 F->hasFnAttribute(Attribute::AlwaysInline))
1347 return false;
1348
1349 // Prohibit function address recording if the function is both internal and
1350 // COMDAT. This avoids the profile data variable referencing internal symbols
1351 // in COMDAT.
1352 if (F->hasLocalLinkage() && F->hasComdat())
1353 return false;
1354
1355 // Check uses of this function for other than direct calls or invokes to it.
1356 // Inline virtual functions have linkeOnceODR linkage. When a key method
1357 // exists, the vtable will only be emitted in the TU where the key method
1358 // is defined. In a TU where vtable is not available, the function won't
1359 // be 'addresstaken'. If its address is not recorded here, the profile data
1360 // with missing address may be picked by the linker leading to missing
1361 // indirect call target info.
1362 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
1363}
1364
1365static inline bool shouldUsePublicSymbol(Function *Fn) {
1366 // It isn't legal to make an alias of this function at all
1367 if (Fn->isDeclarationForLinker())
1368 return true;
1369
1370 // Symbols with local linkage can just use the symbol directly without
1371 // introducing relocations
1372 if (Fn->hasLocalLinkage())
1373 return true;
1374
1375 // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some
1376 // unfavorable interaction between the new alias and the alias renaming done
1377 // in LowerTypeTests under ThinLTO. For comdat functions that would normally
1378 // be deduplicated, but the renaming scheme ends up preventing renaming, since
1379 // it creates unique names for each alias, resulting in duplicated symbols. In
1380 // the future, we should update the CFI related passes to migrate these
1381 // aliases to the same module as the jump-table they refer to will be defined.
1382 if (Fn->hasMetadata(LLVMContext::MD_type))
1383 return true;
1384
1385 // For comdat functions, an alias would need the same linkage as the original
1386 // function and hidden visibility. There is no point in adding an alias with
1387 // identical linkage an visibility to avoid introducing symbolic relocations.
1388 if (Fn->hasComdat() &&
1390 return true;
1391
1392 // its OK to use an alias
1393 return false;
1394}
1395
1397 auto *Int8PtrTy = PointerType::getUnqual(Fn->getContext());
1398 // Store a nullptr in __llvm_profd, if we shouldn't use a real address
1399 if (!shouldRecordFunctionAddr(Fn))
1400 return ConstantPointerNull::get(Int8PtrTy);
1401
1402 // If we can't use an alias, we must use the public symbol, even though this
1403 // may require a symbolic relocation.
1404 if (shouldUsePublicSymbol(Fn))
1405 return Fn;
1406
1407 // When possible use a private alias to avoid symbolic relocations.
1409 Fn->getName() + ".local", Fn);
1410
1411 // When the instrumented function is a COMDAT function, we cannot use a
1412 // private alias. If we did, we would create reference to a local label in
1413 // this function's section. If this version of the function isn't selected by
1414 // the linker, then the metadata would introduce a reference to a discarded
1415 // section. So, for COMDAT functions, we need to adjust the linkage of the
1416 // alias. Using hidden visibility avoids a dynamic relocation and an entry in
1417 // the dynamic symbol table.
1418 //
1419 // Note that this handles COMDAT functions with visibility other than Hidden,
1420 // since that case is covered in shouldUsePublicSymbol()
1421 if (Fn->hasComdat()) {
1422 GA->setLinkage(Fn->getLinkage());
1424 }
1425
1426 // appendToCompilerUsed(*Fn->getParent(), {GA});
1427
1428 return GA;
1429}
1430
1432 // compiler-rt uses linker support to get data/counters/name start/end for
1433 // ELF, COFF, Mach-O and XCOFF.
1434 if (TT.isOSBinFormatELF() || TT.isOSBinFormatCOFF() ||
1435 TT.isOSBinFormatMachO() || TT.isOSBinFormatXCOFF())
1436 return false;
1437
1438 return true;
1439}
1440
1441void InstrLowerer::maybeSetComdat(GlobalVariable *GV, GlobalObject *GO,
1442 StringRef CounterGroupName) {
1443 // Place lowered global variables in a comdat group if the associated function
1444 // or global variable is a COMDAT. This will make sure that only one copy of
1445 // global variable (e.g. function counters) of the COMDAT function will be
1446 // emitted after linking.
1447 bool NeedComdat = needsComdatForCounter(*GO, M);
1448 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
1449
1450 if (!UseComdat)
1451 return;
1452
1453 // Keep in mind that this pass may run before the inliner, so we need to
1454 // create a new comdat group (for counters, profiling data, etc). If we use
1455 // the comdat of the parent function, that will result in relocations against
1456 // discarded sections.
1457 //
1458 // If the data variable is referenced by code, non-counter variables (notably
1459 // profiling data) and counters have to be in different comdats for COFF
1460 // because the Visual C++ linker will report duplicate symbol errors if there
1461 // are multiple external symbols with the same name marked
1462 // IMAGE_COMDAT_SELECT_ASSOCIATIVE.
1463 StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
1464 ? GV->getName()
1465 : CounterGroupName;
1466 Comdat *C = M.getOrInsertComdat(GroupName);
1467
1468 if (!NeedComdat) {
1469 // Object file format must be ELF since `UseComdat && !NeedComdat` is true.
1470 //
1471 // For ELF, when not using COMDAT, put counters, data and values into a
1472 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
1473 // allows -z start-stop-gc to discard the entire group when the function is
1474 // discarded.
1475 C->setSelectionKind(Comdat::NoDeduplicate);
1476 }
1477 GV->setComdat(C);
1478 // COFF doesn't allow the comdat group leader to have private linkage, so
1479 // upgrade private linkage to internal linkage to produce a symbol table
1480 // entry.
1481 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
1483}
1484
1486 if (!profDataReferencedByCode(*GV->getParent()))
1487 return false;
1488
1489 if (!GV->hasLinkOnceLinkage() && !GV->hasLocalLinkage() &&
1491 return true;
1492
1493 // This avoids the profile data from referencing internal symbols in
1494 // COMDAT.
1495 if (GV->hasLocalLinkage() && GV->hasComdat())
1496 return false;
1497
1498 return true;
1499}
1500
1501// FIXME: Introduce an internal alias like what's done for functions to reduce
1502// the number of relocation entries.
1504 auto *Int8PtrTy = PointerType::getUnqual(GV->getContext());
1505
1506 // Store a nullptr in __profvt_ if a real address shouldn't be used.
1507 if (!shouldRecordVTableAddr(GV))
1508 return ConstantPointerNull::get(Int8PtrTy);
1509
1510 return ConstantExpr::getBitCast(GV, Int8PtrTy);
1511}
1512
1513void InstrLowerer::getOrCreateVTableProfData(GlobalVariable *GV) {
1515 "Value profiling is not supported with lightweight instrumentation");
1517 return;
1518
1519 // Skip llvm internal global variable or __prof variables.
1520 if (GV->getName().starts_with("llvm.") ||
1521 GV->getName().starts_with("__llvm") ||
1522 GV->getName().starts_with("__prof"))
1523 return;
1524
1525 // VTableProfData already created
1526 auto It = VTableDataMap.find(GV);
1527 if (It != VTableDataMap.end() && It->second)
1528 return;
1529
1532
1533 // This is to keep consistent with per-function profile data
1534 // for correctness.
1535 if (TT.isOSBinFormatXCOFF()) {
1537 Visibility = GlobalValue::DefaultVisibility;
1538 }
1539
1540 LLVMContext &Ctx = M.getContext();
1541 Type *DataTypes[] = {
1542#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) LLVMType,
1544#undef INSTR_PROF_VTABLE_DATA
1545 };
1546
1547 auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));
1548
1549 // Used by INSTR_PROF_VTABLE_DATA MACRO
1550 Constant *VTableAddr = getVTableAddrForProfData(GV);
1551 const std::string PGOVTableName = getPGOName(*GV);
1552 // Record the length of the vtable. This is needed since vtable pointers
1553 // loaded from C++ objects might be from the middle of a vtable definition.
1554 uint32_t VTableSizeVal =
1555 M.getDataLayout().getTypeAllocSize(GV->getValueType());
1556
1557 Constant *DataVals[] = {
1558#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Init) Init,
1560#undef INSTR_PROF_VTABLE_DATA
1561 };
1562
1563 auto *Data =
1564 new GlobalVariable(M, DataTy, /*constant=*/false, Linkage,
1565 ConstantStruct::get(DataTy, DataVals),
1566 getInstrProfVTableVarPrefix() + PGOVTableName);
1567
1568 Data->setVisibility(Visibility);
1569 Data->setSection(getInstrProfSectionName(IPSK_vtab, TT.getObjectFormat()));
1570 Data->setAlignment(Align(8));
1571
1572 maybeSetComdat(Data, GV, Data->getName());
1573
1574 VTableDataMap[GV] = Data;
1575
1576 ReferencedVTables.push_back(GV);
1577
1578 // VTable <Hash, Addr> is used by runtime but not referenced by other
1579 // sections. Conservatively mark it linker retained.
1580 UsedVars.push_back(Data);
1581}
1582
1583GlobalVariable *InstrLowerer::setupProfileSection(InstrProfInstBase *Inc,
1584 InstrProfSectKind IPSK) {
1585 GlobalVariable *NamePtr = Inc->getName();
1586
1587 // Match the linkage and visibility of the name global.
1588 Function *Fn = Inc->getParent()->getParent();
1590 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1591
1592 // Use internal rather than private linkage so the counter variable shows up
1593 // in the symbol table when using debug info for correlation.
1594 if ((DebugInfoCorrelate ||
1596 TT.isOSBinFormatMachO() && Linkage == GlobalValue::PrivateLinkage)
1598
1599 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1600 // symbols in the same csect won't be discarded. When there are duplicate weak
1601 // symbols, we can NOT guarantee that the relocations get resolved to the
1602 // intended weak symbol, so we can not ensure the correctness of the relative
1603 // CounterPtr, so we have to use private linkage for counter and data symbols.
1604 if (TT.isOSBinFormatXCOFF()) {
1606 Visibility = GlobalValue::DefaultVisibility;
1607 }
1608 // Move the name variable to the right section.
1609 bool Renamed;
1611 StringRef VarPrefix;
1612 std::string VarName;
1613 if (IPSK == IPSK_cnts) {
1614 VarPrefix = getInstrProfCountersVarPrefix();
1615 VarName = getVarName(Inc, VarPrefix, Renamed);
1616 InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Inc);
1617 Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);
1618 } else if (IPSK == IPSK_bitmap) {
1619 VarPrefix = getInstrProfBitmapVarPrefix();
1620 VarName = getVarName(Inc, VarPrefix, Renamed);
1621 InstrProfMCDCBitmapInstBase *BitmapUpdate =
1622 dyn_cast<InstrProfMCDCBitmapInstBase>(Inc);
1623 Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);
1624 } else {
1625 llvm_unreachable("Profile Section must be for Counters or Bitmaps");
1626 }
1627
1628 Ptr->setVisibility(Visibility);
1629 // Put the counters and bitmaps in their own sections so linkers can
1630 // remove unneeded sections.
1631 Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat()));
1632 Ptr->setLinkage(Linkage);
1633 maybeSetComdat(Ptr, Fn, VarName);
1634 return Ptr;
1635}
1636
1638InstrLowerer::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
1640 GlobalValue::LinkageTypes Linkage) {
1641 uint64_t NumBytes = Inc->getNumBitmapBytes();
1642 auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M.getContext()), NumBytes);
1643 auto GV = new GlobalVariable(M, BitmapTy, false, Linkage,
1644 Constant::getNullValue(BitmapTy), Name);
1645 GV->setAlignment(Align(1));
1646 return GV;
1647}
1648
1650InstrLowerer::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
1651 GlobalVariable *NamePtr = Inc->getName();
1652 auto &PD = ProfileDataMap[NamePtr];
1653 if (PD.RegionBitmaps)
1654 return PD.RegionBitmaps;
1655
1656 // If RegionBitmaps doesn't already exist, create it by first setting up
1657 // the corresponding profile section.
1658 auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
1659 PD.RegionBitmaps = BitmapPtr;
1660 PD.NumBitmapBytes = Inc->getNumBitmapBytes();
1661 return PD.RegionBitmaps;
1662}
1663
1665InstrLowerer::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
1666 GlobalValue::LinkageTypes Linkage) {
1667 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1668 auto &Ctx = M.getContext();
1669 GlobalVariable *GV;
1670 if (isa<InstrProfCoverInst>(Inc)) {
1671 auto *CounterTy = Type::getInt8Ty(Ctx);
1672 auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);
1673 // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
1674 std::vector<Constant *> InitialValues(NumCounters,
1675 Constant::getAllOnesValue(CounterTy));
1676 GV = new GlobalVariable(M, CounterArrTy, false, Linkage,
1677 ConstantArray::get(CounterArrTy, InitialValues),
1678 Name);
1679 GV->setAlignment(Align(1));
1680 } else {
1681 auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
1682 GV = new GlobalVariable(M, CounterTy, false, Linkage,
1683 Constant::getNullValue(CounterTy), Name);
1684 GV->setAlignment(Align(8));
1685 }
1686 return GV;
1687}
1688
1690InstrLowerer::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
1691 GlobalVariable *NamePtr = Inc->getName();
1692 auto &PD = ProfileDataMap[NamePtr];
1693 if (PD.RegionCounters)
1694 return PD.RegionCounters;
1695
1696 // If RegionCounters doesn't already exist, create it by first setting up
1697 // the corresponding profile section.
1698 auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
1699 PD.RegionCounters = CounterPtr;
1700
1701 if (DebugInfoCorrelate ||
1703 LLVMContext &Ctx = M.getContext();
1704 Function *Fn = Inc->getParent()->getParent();
1705 if (auto *SP = Fn->getSubprogram()) {
1706 DIBuilder DB(M, true, SP->getUnit());
1707 Metadata *FunctionNameAnnotation[] = {
1710 };
1711 Metadata *CFGHashAnnotation[] = {
1714 };
1715 Metadata *NumCountersAnnotation[] = {
1718 };
1719 auto Annotations = DB.getOrCreateArray({
1720 MDNode::get(Ctx, FunctionNameAnnotation),
1721 MDNode::get(Ctx, CFGHashAnnotation),
1722 MDNode::get(Ctx, NumCountersAnnotation),
1723 });
1724 auto *DICounter = DB.createGlobalVariableExpression(
1725 SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
1726 /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),
1727 CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
1728 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1729 Annotations);
1730 CounterPtr->addDebugInfo(DICounter);
1731 DB.finalize();
1732 }
1733
1734 // Mark the counter variable as used so that it isn't optimized out.
1735 CompilerUsedVars.push_back(PD.RegionCounters);
1736 }
1737
1738 // Create the data variable (if it doesn't already exist).
1739 createDataVariable(Inc);
1740
1741 return PD.RegionCounters;
1742}
1743
1744void InstrLowerer::createDataVariable(InstrProfCntrInstBase *Inc) {
1745 // When debug information is correlated to profile data, a data variable
1746 // is not needed.
1748 return;
1749
1750 GlobalVariable *NamePtr = Inc->getName();
1751 auto &PD = ProfileDataMap[NamePtr];
1752
1753 // Return if data variable was already created.
1754 if (PD.DataVar)
1755 return;
1756
1757 LLVMContext &Ctx = M.getContext();
1758
1759 Function *Fn = Inc->getParent()->getParent();
1761 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1762
1763 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1764 // symbols in the same csect won't be discarded. When there are duplicate weak
1765 // symbols, we can NOT guarantee that the relocations get resolved to the
1766 // intended weak symbol, so we can not ensure the correctness of the relative
1767 // CounterPtr, so we have to use private linkage for counter and data symbols.
1768 if (TT.isOSBinFormatXCOFF()) {
1770 Visibility = GlobalValue::DefaultVisibility;
1771 }
1772
1773 bool NeedComdat = needsComdatForCounter(*Fn, M);
1774 bool Renamed;
1775
1776 // The Data Variable section is anchored to profile counters.
1777 std::string CntsVarName =
1779 std::string DataVarName =
1780 getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
1781
1782 auto *Int8PtrTy = PointerType::getUnqual(Ctx);
1783 // Allocate statically the array of pointers to value profile nodes for
1784 // the current function.
1785 Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
1786 uint64_t NS = 0;
1787 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1788 NS += PD.NumValueSites[Kind];
1789 if (NS > 0 && ValueProfileStaticAlloc &&
1791 ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
1792 auto *ValuesVar = new GlobalVariable(
1793 M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
1794 getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
1795 ValuesVar->setVisibility(Visibility);
1796 setGlobalVariableLargeSection(TT, *ValuesVar);
1797 ValuesVar->setSection(
1798 getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
1799 ValuesVar->setAlignment(Align(8));
1800 maybeSetComdat(ValuesVar, Fn, CntsVarName);
1801 ValuesPtrExpr = ValuesVar;
1802 }
1803
1804 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1805 auto *CounterPtr = PD.RegionCounters;
1806
1807 uint64_t NumBitmapBytes = PD.NumBitmapBytes;
1808
1809 // Create data variable.
1810 auto *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext());
1811 auto *Int16Ty = Type::getInt16Ty(Ctx);
1812 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
1813 Type *DataTypes[] = {
1814#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1816 };
1817 auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));
1818
1819 Constant *FunctionAddr = getFuncAddrForProfData(Fn);
1820
1821 Constant *Int16ArrayVals[IPVK_Last + 1];
1822 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1823 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
1824
1825 // If the data variable is not referenced by code (if we don't emit
1826 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
1827 // data variable live under linker GC, the data variable can be private. This
1828 // optimization applies to ELF.
1829 //
1830 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
1831 // to be false.
1832 //
1833 // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
1834 // that other copies must have the same CFG and cannot have value profiling.
1835 // If no hash suffix, other profd copies may be referenced by code.
1836 if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1837 (TT.isOSBinFormatELF() ||
1838 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
1840 Visibility = GlobalValue::DefaultVisibility;
1841 }
1842 auto *Data =
1843 new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName);
1844 Constant *RelativeCounterPtr;
1845 GlobalVariable *BitmapPtr = PD.RegionBitmaps;
1846 Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);
1847 InstrProfSectKind DataSectionKind;
1848 // With binary profile correlation, profile data is not loaded into memory.
1849 // profile data must reference profile counter with an absolute relocation.
1851 DataSectionKind = IPSK_covdata;
1852 RelativeCounterPtr = ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy);
1853 if (BitmapPtr != nullptr)
1854 RelativeBitmapPtr = ConstantExpr::getPtrToInt(BitmapPtr, IntPtrTy);
1855 } else {
1856 // Reference the counter variable with a label difference (link-time
1857 // constant).
1858 DataSectionKind = IPSK_data;
1859 RelativeCounterPtr =
1860 ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
1861 ConstantExpr::getPtrToInt(Data, IntPtrTy));
1862 if (BitmapPtr != nullptr)
1863 RelativeBitmapPtr =
1865 ConstantExpr::getPtrToInt(Data, IntPtrTy));
1866 }
1867
1868 Constant *DataVals[] = {
1869#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1871 };
1872 Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
1873
1874 Data->setVisibility(Visibility);
1875 Data->setSection(
1876 getInstrProfSectionName(DataSectionKind, TT.getObjectFormat()));
1877 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
1878 maybeSetComdat(Data, Fn, CntsVarName);
1879
1880 PD.DataVar = Data;
1881
1882 // Mark the data variable as used so that it isn't stripped out.
1883 CompilerUsedVars.push_back(Data);
1884 // Now that the linkage set by the FE has been passed to the data and counter
1885 // variables, reset Name variable's linkage and visibility to private so that
1886 // it can be removed later by the compiler.
1888 // Collect the referenced names to be used by emitNameData.
1889 ReferencedNames.push_back(NamePtr);
1890}
1891
1892void InstrLowerer::emitVNodes() {
1893 if (!ValueProfileStaticAlloc)
1894 return;
1895
1896 // For now only support this on platforms that do
1897 // not require runtime registration to discover
1898 // named section start/end.
1900 return;
1901
1902 size_t TotalNS = 0;
1903 for (auto &PD : ProfileDataMap) {
1904 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1905 TotalNS += PD.second.NumValueSites[Kind];
1906 }
1907
1908 if (!TotalNS)
1909 return;
1910
1911 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1912// Heuristic for small programs with very few total value sites.
1913// The default value of vp-counters-per-site is chosen based on
1914// the observation that large apps usually have a low percentage
1915// of value sites that actually have any profile data, and thus
1916// the average number of counters per site is low. For small
1917// apps with very few sites, this may not be true. Bump up the
1918// number of counters in this case.
1919#define INSTR_PROF_MIN_VAL_COUNTS 10
1920 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
1921 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
1922
1923 auto &Ctx = M.getContext();
1924 Type *VNodeTypes[] = {
1925#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1927 };
1928 auto *VNodeTy = StructType::get(Ctx, ArrayRef(VNodeTypes));
1929
1930 ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
1931 auto *VNodesVar = new GlobalVariable(
1932 M, VNodesTy, false, GlobalValue::PrivateLinkage,
1934 setGlobalVariableLargeSection(TT, *VNodesVar);
1935 VNodesVar->setSection(
1936 getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
1937 VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(VNodesTy));
1938 // VNodesVar is used by runtime but not referenced via relocation by other
1939 // sections. Conservatively make it linker retained.
1940 UsedVars.push_back(VNodesVar);
1941}
1942
1943void InstrLowerer::emitNameData() {
1944 std::string UncompressedData;
1945
1946 if (ReferencedNames.empty())
1947 return;
1948
1949 std::string CompressedNameStr;
1950 if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
1952 report_fatal_error(Twine(toString(std::move(E))), false);
1953 }
1954
1955 auto &Ctx = M.getContext();
1956 auto *NamesVal =
1957 ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false);
1958 NamesVar = new GlobalVariable(M, NamesVal->getType(), true,
1961 NamesSize = CompressedNameStr.size();
1962 setGlobalVariableLargeSection(TT, *NamesVar);
1963 NamesVar->setSection(
1965 ? getInstrProfSectionName(IPSK_covname, TT.getObjectFormat())
1966 : getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
1967 // On COFF, it's important to reduce the alignment down to 1 to prevent the
1968 // linker from inserting padding before the start of the names section or
1969 // between names entries.
1970 NamesVar->setAlignment(Align(1));
1971 // NamesVar is used by runtime but not referenced via relocation by other
1972 // sections. Conservatively make it linker retained.
1973 UsedVars.push_back(NamesVar);
1974
1975 for (auto *NamePtr : ReferencedNames)
1976 NamePtr->eraseFromParent();
1977}
1978
1979void InstrLowerer::emitVTableNames() {
1980 if (!EnableVTableValueProfiling || ReferencedVTables.empty())
1981 return;
1982
1983 // Collect the PGO names of referenced vtables and compress them.
1984 std::string CompressedVTableNames;
1985 if (Error E = collectVTableStrings(ReferencedVTables, CompressedVTableNames,
1987 report_fatal_error(Twine(toString(std::move(E))), false);
1988 }
1989
1990 auto &Ctx = M.getContext();
1991 auto *VTableNamesVal = ConstantDataArray::getString(
1992 Ctx, StringRef(CompressedVTableNames), false /* AddNull */);
1993 GlobalVariable *VTableNamesVar =
1994 new GlobalVariable(M, VTableNamesVal->getType(), true /* constant */,
1995 GlobalValue::PrivateLinkage, VTableNamesVal,
1997 VTableNamesVar->setSection(
1998 getInstrProfSectionName(IPSK_vname, TT.getObjectFormat()));
1999 VTableNamesVar->setAlignment(Align(1));
2000 // Make VTableNames linker retained.
2001 UsedVars.push_back(VTableNamesVar);
2002}
2003
2004void InstrLowerer::emitRegistration() {
2006 return;
2007
2008 // Construct the function.
2009 auto *VoidTy = Type::getVoidTy(M.getContext());
2010 auto *VoidPtrTy = PointerType::getUnqual(M.getContext());
2011 auto *Int64Ty = Type::getInt64Ty(M.getContext());
2012 auto *RegisterFTy = FunctionType::get(VoidTy, false);
2013 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
2015 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2016 if (Options.NoRedZone)
2017 RegisterF->addFnAttr(Attribute::NoRedZone);
2018
2019 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
2020 auto *RuntimeRegisterF =
2023
2024 IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", RegisterF));
2025 for (Value *Data : CompilerUsedVars)
2026 if (!isa<Function>(Data))
2027 IRB.CreateCall(RuntimeRegisterF, Data);
2028 for (Value *Data : UsedVars)
2029 if (Data != NamesVar && !isa<Function>(Data))
2030 IRB.CreateCall(RuntimeRegisterF, Data);
2031
2032 if (NamesVar) {
2033 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
2034 auto *NamesRegisterTy =
2035 FunctionType::get(VoidTy, ArrayRef(ParamTypes), false);
2036 auto *NamesRegisterF =
2039 IRB.CreateCall(NamesRegisterF, {NamesVar, IRB.getInt64(NamesSize)});
2040 }
2041
2042 IRB.CreateRetVoid();
2043}
2044
2045bool InstrLowerer::emitRuntimeHook() {
2046 // We expect the linker to be invoked with -u<hook_var> flag for Linux
2047 // in which case there is no need to emit the external variable.
2048 if (TT.isOSLinux() || TT.isOSAIX())
2049 return false;
2050
2051 // If the module's provided its own runtime, we don't need to do anything.
2052 if (M.getGlobalVariable(getInstrProfRuntimeHookVarName()))
2053 return false;
2054
2055 // Declare an external variable that will pull in the runtime initialization.
2056 auto *Int32Ty = Type::getInt32Ty(M.getContext());
2057 auto *Var =
2058 new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage,
2060 Var->setVisibility(GlobalValue::HiddenVisibility);
2061
2062 if (TT.isOSBinFormatELF() && !TT.isPS()) {
2063 // Mark the user variable as used so that it isn't stripped out.
2064 CompilerUsedVars.push_back(Var);
2065 } else {
2066 // Make a function that uses it.
2067 auto *User = Function::Create(FunctionType::get(Int32Ty, false),
2070 User->addFnAttr(Attribute::NoInline);
2071 if (Options.NoRedZone)
2072 User->addFnAttr(Attribute::NoRedZone);
2073 User->setVisibility(GlobalValue::HiddenVisibility);
2074 if (TT.supportsCOMDAT())
2075 User->setComdat(M.getOrInsertComdat(User->getName()));
2076
2077 IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", User));
2078 auto *Load = IRB.CreateLoad(Int32Ty, Var);
2079 IRB.CreateRet(Load);
2080
2081 // Mark the function as used so that it isn't stripped out.
2082 CompilerUsedVars.push_back(User);
2083 }
2084 return true;
2085}
2086
2087void InstrLowerer::emitUses() {
2088 // The metadata sections are parallel arrays. Optimizers (e.g.
2089 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
2090 // we conservatively retain all unconditionally in the compiler.
2091 //
2092 // On ELF and Mach-O, the linker can guarantee the associated sections will be
2093 // retained or discarded as a unit, so llvm.compiler.used is sufficient.
2094 // Similarly on COFF, if prof data is not referenced by code we use one comdat
2095 // and ensure this GC property as well. Otherwise, we have to conservatively
2096 // make all of the sections retained by the linker.
2097 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
2098 (TT.isOSBinFormatCOFF() && !DataReferencedByCode))
2099 appendToCompilerUsed(M, CompilerUsedVars);
2100 else
2101 appendToUsed(M, CompilerUsedVars);
2102
2103 // We do not add proper references from used metadata sections to NamesVar and
2104 // VNodesVar, so we have to be conservative and place them in llvm.used
2105 // regardless of the target,
2106 appendToUsed(M, UsedVars);
2107}
2108
2109void InstrLowerer::emitInitialization() {
2110 // Create ProfileFileName variable. Don't don't this for the
2111 // context-sensitive instrumentation lowering: This lowering is after
2112 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
2113 // have already create the variable before LTO/ThinLTO linking.
2114 if (!IsCS)
2115 createProfileFileNameVar(M, Options.InstrProfileOutput);
2116 Function *RegisterF = M.getFunction(getInstrProfRegFuncsName());
2117 if (!RegisterF)
2118 return;
2119
2120 // Create the initialization function.
2121 auto *VoidTy = Type::getVoidTy(M.getContext());
2122 auto *F = Function::Create(FunctionType::get(VoidTy, false),
2125 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
2126 F->addFnAttr(Attribute::NoInline);
2127 if (Options.NoRedZone)
2128 F->addFnAttr(Attribute::NoRedZone);
2129
2130 // Add the basic block and the necessary calls.
2131 IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", F));
2132 IRB.CreateCall(RegisterF, {});
2133 IRB.CreateRetVoid();
2134
2135 appendToGlobalCtors(M, F, 0);
2136}
2137
2138namespace llvm {
2139// Create the variable for profile sampling.
2141 const StringRef VarName(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SAMPLING_VAR));
2142 IntegerType *SamplingVarTy;
2143 Constant *ValueZero;
2144 if (SampledInstrPeriod.getValue() <= USHRT_MAX) {
2145 SamplingVarTy = Type::getInt16Ty(M.getContext());
2146 ValueZero = Constant::getIntegerValue(SamplingVarTy, APInt(16, 0));
2147 } else {
2148 SamplingVarTy = Type::getInt32Ty(M.getContext());
2149 ValueZero = Constant::getIntegerValue(SamplingVarTy, APInt(32, 0));
2150 }
2151 auto SamplingVar = new GlobalVariable(
2152 M, SamplingVarTy, false, GlobalValue::WeakAnyLinkage, ValueZero, VarName);
2153 SamplingVar->setVisibility(GlobalValue::DefaultVisibility);
2154 SamplingVar->setThreadLocal(true);
2155 Triple TT(M.getTargetTriple());
2156 if (TT.supportsCOMDAT()) {
2157 SamplingVar->setLinkage(GlobalValue::ExternalLinkage);
2158 SamplingVar->setComdat(M.getOrInsertComdat(VarName));
2159 }
2160 appendToCompilerUsed(M, SamplingVar);
2161}
2162} // namespace llvm
This file contains the simple types necessary to represent the attributes associated with functions a...
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
Definition: CommandLine.h:686
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static unsigned InstrCount
static bool lowerIntrinsics(Module &M)
#define LLVM_DEBUG(X)
Definition: Debug.h:101
@ Default
Definition: DwarfDebug.cpp:87
uint64_t Addr
std::string Name
Hexagon Hardware Loops
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static bool shouldRecordVTableAddr(GlobalVariable *GV)
static bool shouldRecordFunctionAddr(Function *F)
static bool needsRuntimeHookUnconditionally(const Triple &TT)
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
#define INSTR_PROF_MIN_VAL_COUNTS
static Constant * getFuncAddrForProfData(Function *Fn)
static bool shouldUsePublicSymbol(Function *Fn)
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
static Constant * getVTableAddrForProfData(GlobalVariable *GV)
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
static LVOptions Options
Definition: LVOptions.cpp:25
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Memory SSA
Definition: MemorySSA.cpp:72
Module.h This file contains the declarations for the Module class.
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
if(VerifyEach)
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
Class for arbitrary precision integers.
Definition: APInt.h:78
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:253
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:405
Annotations lets you mark points and ranges inside source code, for tests:
Definition: Annotations.h:53
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:635
@ Add
*p = old + v
Definition: Instructions.h:712
@ Or
*p = old | v
Definition: Instructions.h:720
LLVM Basic Block Representation.
Definition: BasicBlock.h:61
iterator end()
Definition: BasicBlock.h:461
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:448
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:416
const Instruction & front() const
Definition: BasicBlock.h:471
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:212
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:219
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
This class represents a function call, abstracting a target machine's calling convention.
static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)
@ NoDeduplicate
No deduplication is performed.
Definition: Comdat.h:39
ConstantArray - Constant Array Declarations.
Definition: Constants.h:424
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1292
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:528
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2950
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2618
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2267
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2295
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:155
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1800
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1357
This is an important base class in LLVM.
Definition: Constant.h:42
static Constant * getIntegerValue(Type *Ty, const APInt &V)
Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...
Definition: Constants.cpp:400
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:417
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:370
bool isZeroValue() const
Return true if the value is negative zero or null value.
Definition: Constants.cpp:76
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:168
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
void addFnAttr(Attribute::AttrKind Kind)
Add function attributes to this function.
Definition: Function.cpp:653
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:172
const BasicBlock & getEntryBlock() const
Definition: Function.h:807
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1837
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:380
Argument * getArg(unsigned i) const
Definition: Function.h:884
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
Definition: Globals.cpp:550
bool hasMetadata() const
Return true if this value has any metadata attached to it.
Definition: Value.h:589
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
Definition: Globals.cpp:137
void setComdat(Comdat *C)
Definition: Globals.cpp:206
bool hasComdat() const
Definition: GlobalObject.h:128
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:267
bool hasLinkOnceLinkage() const
Definition: GlobalValue.h:515
VisibilityTypes getVisibility() const
Definition: GlobalValue.h:248
bool isDeclaration() const
Return true if the primary definition of this global value is outside of the current translation unit...
Definition: Globals.cpp:290
LinkageTypes getLinkage() const
Definition: GlobalValue.h:546
bool hasLocalLinkage() const
Definition: GlobalValue.h:528
bool hasPrivateLinkage() const
Definition: GlobalValue.h:527
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:537
bool isDeclarationForLinker() const
Definition: GlobalValue.h:618
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:656
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:66
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:67
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:68
void setVisibility(VisibilityTypes V)
Definition: GlobalValue.h:254
bool hasAvailableExternallyLinkage() const
Definition: GlobalValue.h:512
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:51
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:60
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:59
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:52
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:56
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:55
Type * getValueType() const
Definition: GlobalValue.h:296
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:481
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2130
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1442
ConstantInt * getInt8(uint8_t C)
Get a constant 8-bit value.
Definition: IRBuilder.h:473
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", GEPNoWrapFlags NW=GEPNoWrapFlags::none())
Definition: IRBuilder.h:1984
Value * CreateICmpNE(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2253
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:483
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Definition: IRBuilder.h:142
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Definition: IRBuilder.h:1125
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition: IRBuilder.h:1795
Value * CreateShl(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1421
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1480
ReturnInst * CreateRetVoid()
Create a 'ret void' instruction.
Definition: IRBuilder.h:1095
Value * CreateConstInBoundsGEP2_32(Type *Ty, Value *Ptr, unsigned Idx0, unsigned Idx1, const Twine &Name="")
Definition: IRBuilder.h:1918
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1808
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1332
Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:2125
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, MaybeAlign Align, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Definition: IRBuilder.h:1859
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Definition: IRBuilder.h:2015
Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1502
ConstantInt * getInt16(uint16_t C)
Get a constant 16-bit value.
Definition: IRBuilder.h:478
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2420
Value * CreateInBoundsPtrAdd(Value *Ptr, Value *Offset, const Twine &Name="")
Definition: IRBuilder.h:1989
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2674
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:563
A base class for all instrprof counter intrinsics.
ConstantInt * getIndex() const
ConstantInt * getNumCounters() const
static const char * FunctionNameAttributeName
static const char * CFGHashAttributeName
static const char * NumCountersAttributeName
This represents the llvm.instrprof.cover intrinsic.
This represents the llvm.instrprof.increment intrinsic.
A base class for all instrprof intrinsics.
GlobalVariable * getName() const
ConstantInt * getHash() const
A base class for instrprof mcdc intrinsics that require global bitmap bytes.
This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic.
ConstantInt * getBitmapIndex() const
This represents the llvm.instrprof.timestamp intrinsic.
This represents the llvm.instrprof.value.profile intrinsic.
ConstantInt * getIndex() const
ConstantInt * getValueKind() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:92
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:70
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1642
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
Class to represent integer types.
Definition: DerivedTypes.h:40
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
Helper class for promoting a collection of loads and stores into SSA Form using the SSAUpdater.
Definition: SSAUpdater.h:151
virtual void doExtraRewritesBeforeFinalDeletion()
This hook is invoked after all the stores are found and inserted as available values.
Definition: SSAUpdater.h:176
An instruction for reading from memory.
Definition: Instructions.h:174
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:44
MDNode * createUnlikelyBranchWeights()
Return metadata containing two branch weights, with significant bias towards false destination.
Definition: MDBuilder.cpp:47
Metadata node.
Definition: Metadata.h:1069
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1542
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:606
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
A set of analyses that are preserved following a run of a transformation pass.
Definition: Analysis.h:111
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: Analysis.h:114
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: Analysis.h:117
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition: SSAUpdater.h:40
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:368
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:503
size_t size() const
Definition: SmallVector.h:92
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:587
void push_back(const T &Elt)
Definition: SmallVector.h:427
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1210
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:556
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition: StringRef.h:250
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:361
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:128
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
Definition: User.h:169
unsigned getNumOperands() const
Definition: User.h:191
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
LLVMContext & getContext() const
All values hold a context through their type.
Definition: Value.cpp:1075
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
const ParentTy * getParent() const
Definition: ilist_node.h:32
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
ValueKind
Value kinds.
@ Exit
Definition: COFF.h:827
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:1096
@ PD
PD - Prefix code for packed double precision vector floating point operations performed in the SSE re...
Definition: X86BaseInfo.h:721
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
Definition: CommandLine.h:711
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:92
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:163
void createProfileSamplingVar(Module &M)
StringRef getInstrProfBitmapVarPrefix()
Return the name prefix of profile bitmap variables.
Definition: InstrProf.h:104
cl::opt< bool > DoInstrProfNameCompression
cl::opt< InstrProfCorrelator::ProfCorrelatorKind > ProfileCorrelate("profile-correlate", cl::desc("Use debug info or binary file to correlate profiles."), cl::init(InstrProfCorrelator::NONE), cl::values(clEnumValN(InstrProfCorrelator::NONE, "", "No profile correlation"), clEnumValN(InstrProfCorrelator::DEBUG_INFO, "debug-info", "Use debug info to correlate"), clEnumValN(InstrProfCorrelator::BINARY, "binary", "Use binary to correlate")))
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:656
StringRef getInstrProfVTableNamesVarName()
Definition: InstrProf.h:116
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:98
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:129
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:236
cl::opt< bool > DebugInfoCorrelate
bool needsComdatForCounter(const GlobalObject &GV, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1416
std::string getPGOName(const GlobalVariable &V, bool InLTO=false)
Definition: InstrProf.cpp:395
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:158
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:107
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1729
StringRef getInstrProfCounterBiasVarName()
Definition: InstrProf.h:173
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:419
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition: InstrProf.h:169
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:138
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
Definition: InstrProf.cpp:707
InstrProfSectKind
Definition: InstrProf.h:60
void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:101
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1736
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:167
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:700
StringRef getInstrProfBitmapBiasVarName()
Definition: InstrProf.h:177
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition: InstrProf.h:87
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:150
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
@ Add
Sum of integers.
Error collectVTableStrings(ArrayRef< GlobalVariable * > VTables, std::string &Result, bool doCompression)
Definition: InstrProf.cpp:717
void setGlobalVariableLargeSection(const Triple &TargetTriple, GlobalVariable &GV)
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1464
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1487
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:74
bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src, const BasicBlock &Dest)
auto predecessors(const MachineBasicBlock *BB)
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:81
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:144
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition: InstrProf.h:114
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1442
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:110
cl::opt< bool > EnableVTableValueProfiling("enable-vtable-value-profiling", cl::init(false), cl::desc("If true, the virtual table address will be instrumented to know " "the types of a C++ pointer. The information is used in indirect " "call promotion to do selective vtable-based comparison."))
StringRef getInstrProfVTableVarPrefix()
Return the name prefix of variables containing virtual table profile data.
Definition: InstrProf.h:95
#define NC
Definition: regutils.h:42
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Options for the frontend instrumentation based profiling pass.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117