LLVM 18.0.0git
InstrProfiling.cpp
Go to the documentation of this file.
1//===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This pass lowers instrprof_* intrinsics emitted by an instrumentor.
10// It also builds the data structures and initialization code needed for
11// updating execution counts and emitting the profile at runtime.
12//
13//===----------------------------------------------------------------------===//
14
16#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/STLExtras.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/Twine.h"
25#include "llvm/IR/Attributes.h"
26#include "llvm/IR/BasicBlock.h"
27#include "llvm/IR/CFG.h"
28#include "llvm/IR/Constant.h"
29#include "llvm/IR/Constants.h"
30#include "llvm/IR/DIBuilder.h"
33#include "llvm/IR/Dominators.h"
34#include "llvm/IR/Function.h"
35#include "llvm/IR/GlobalValue.h"
37#include "llvm/IR/IRBuilder.h"
38#include "llvm/IR/Instruction.h"
41#include "llvm/IR/Module.h"
42#include "llvm/IR/Type.h"
44#include "llvm/Pass.h"
49#include "llvm/Support/Error.h"
56#include <algorithm>
57#include <cassert>
58#include <cstdint>
59#include <string>
60
61using namespace llvm;
62
63#define DEBUG_TYPE "instrprof"
64
65namespace llvm {
67 DebugInfoCorrelate("debug-info-correlate",
68 cl::desc("Use debug info to correlate profiles."),
69 cl::init(false));
70} // namespace llvm
71
72namespace {
73
74cl::opt<bool> DoHashBasedCounterSplit(
75 "hash-based-counter-split",
76 cl::desc("Rename counter variable of a comdat function based on cfg hash"),
77 cl::init(true));
78
80 RuntimeCounterRelocation("runtime-counter-relocation",
81 cl::desc("Enable relocating counters at runtime."),
82 cl::init(false));
83
84cl::opt<bool> ValueProfileStaticAlloc(
85 "vp-static-alloc",
86 cl::desc("Do static counter allocation for value profiler"),
87 cl::init(true));
88
89cl::opt<double> NumCountersPerValueSite(
90 "vp-counters-per-site",
91 cl::desc("The average number of profile counters allocated "
92 "per value profiling site."),
93 // This is set to a very small value because in real programs, only
94 // a very small percentage of value sites have non-zero targets, e.g, 1/30.
95 // For those sites with non-zero profile, the average number of targets
96 // is usually smaller than 2.
97 cl::init(1.0));
98
99cl::opt<bool> AtomicCounterUpdateAll(
100 "instrprof-atomic-counter-update-all",
101 cl::desc("Make all profile counter updates atomic (for testing only)"),
102 cl::init(false));
103
104cl::opt<bool> AtomicCounterUpdatePromoted(
105 "atomic-counter-update-promoted",
106 cl::desc("Do counter update using atomic fetch add "
107 " for promoted counters only"),
108 cl::init(false));
109
110cl::opt<bool> AtomicFirstCounter(
111 "atomic-first-counter",
112 cl::desc("Use atomic fetch add for first counter in a function (usually "
113 "the entry counter)"),
114 cl::init(false));
115
116// If the option is not specified, the default behavior about whether
117// counter promotion is done depends on how instrumentaiton lowering
118// pipeline is setup, i.e., the default value of true of this option
119// does not mean the promotion will be done by default. Explicitly
120// setting this option can override the default behavior.
121cl::opt<bool> DoCounterPromotion("do-counter-promotion",
122 cl::desc("Do counter register promotion"),
123 cl::init(false));
124cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
125 "max-counter-promotions-per-loop", cl::init(20),
126 cl::desc("Max number counter promotions per loop to avoid"
127 " increasing register pressure too much"));
128
129// A debug option
131 MaxNumOfPromotions("max-counter-promotions", cl::init(-1),
132 cl::desc("Max number of allowed counter promotions"));
133
134cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
135 "speculative-counter-promotion-max-exiting", cl::init(3),
136 cl::desc("The max number of exiting blocks of a loop to allow "
137 " speculative counter promotion"));
138
139cl::opt<bool> SpeculativeCounterPromotionToLoop(
140 "speculative-counter-promotion-to-loop",
141 cl::desc("When the option is false, if the target block is in a loop, "
142 "the promotion will be disallowed unless the promoted counter "
143 " update can be further/iteratively promoted into an acyclic "
144 " region."));
145
146cl::opt<bool> IterativeCounterPromotion(
147 "iterative-counter-promotion", cl::init(true),
148 cl::desc("Allow counter promotion across the whole loop nest."));
149
150cl::opt<bool> SkipRetExitBlock(
151 "skip-ret-exit-block", cl::init(true),
152 cl::desc("Suppress counter promotion if exit blocks contain ret."));
153
154///
155/// A helper class to promote one counter RMW operation in the loop
156/// into register update.
157///
158/// RWM update for the counter will be sinked out of the loop after
159/// the transformation.
160///
161class PGOCounterPromoterHelper : public LoadAndStorePromoter {
162public:
163 PGOCounterPromoterHelper(
165 BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
166 ArrayRef<Instruction *> InsertPts,
168 LoopInfo &LI)
169 : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
170 InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
171 assert(isa<LoadInst>(L));
172 assert(isa<StoreInst>(S));
173 SSA.AddAvailableValue(PH, Init);
174 }
175
176 void doExtraRewritesBeforeFinalDeletion() override {
177 for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
178 BasicBlock *ExitBlock = ExitBlocks[i];
179 Instruction *InsertPos = InsertPts[i];
180 // Get LiveIn value into the ExitBlock. If there are multiple
181 // predecessors, the value is defined by a PHI node in this
182 // block.
183 Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
184 Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
185 Type *Ty = LiveInValue->getType();
186 IRBuilder<> Builder(InsertPos);
187 if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Addr)) {
188 // If isRuntimeCounterRelocationEnabled() is true then the address of
189 // the store instruction is computed with two instructions in
190 // InstrProfiling::getCounterAddress(). We need to copy those
191 // instructions to this block to compute Addr correctly.
192 // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
193 // %Addr = inttoptr i64 %BiasAdd to i64*
194 auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
195 assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
196 Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
197 Addr = Builder.CreateIntToPtr(BiasInst,
198 PointerType::getUnqual(Ty->getContext()));
199 }
200 if (AtomicCounterUpdatePromoted)
201 // automic update currently can only be promoted across the current
202 // loop, not the whole loop nest.
203 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
204 MaybeAlign(),
205 AtomicOrdering::SequentiallyConsistent);
206 else {
207 LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
208 auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
209 auto *NewStore = Builder.CreateStore(NewVal, Addr);
210
211 // Now update the parent loop's candidate list:
212 if (IterativeCounterPromotion) {
213 auto *TargetLoop = LI.getLoopFor(ExitBlock);
214 if (TargetLoop)
215 LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
216 }
217 }
218 }
219 }
220
221private:
223 ArrayRef<BasicBlock *> ExitBlocks;
224 ArrayRef<Instruction *> InsertPts;
226 LoopInfo &LI;
227};
228
229/// A helper class to do register promotion for all profile counter
230/// updates in a loop.
231///
232class PGOCounterPromoter {
233public:
234 PGOCounterPromoter(
236 Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
237 : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {
238
239 // Skip collection of ExitBlocks and InsertPts for loops that will not be
240 // able to have counters promoted.
241 SmallVector<BasicBlock *, 8> LoopExitBlocks;
243
244 L.getExitBlocks(LoopExitBlocks);
245 if (!isPromotionPossible(&L, LoopExitBlocks))
246 return;
247
248 for (BasicBlock *ExitBlock : LoopExitBlocks) {
249 if (BlockSet.insert(ExitBlock).second &&
250 llvm::none_of(predecessors(ExitBlock), [&](const BasicBlock *Pred) {
251 return llvm::isPresplitCoroSuspendExitEdge(*Pred, *ExitBlock);
252 })) {
253 ExitBlocks.push_back(ExitBlock);
254 InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
255 }
256 }
257 }
258
259 bool run(int64_t *NumPromoted) {
260 // Skip 'infinite' loops:
261 if (ExitBlocks.size() == 0)
262 return false;
263
264 // Skip if any of the ExitBlocks contains a ret instruction.
265 // This is to prevent dumping of incomplete profile -- if the
266 // the loop is a long running loop and dump is called in the middle
267 // of the loop, the result profile is incomplete.
268 // FIXME: add other heuristics to detect long running loops.
269 if (SkipRetExitBlock) {
270 for (auto *BB : ExitBlocks)
271 if (isa<ReturnInst>(BB->getTerminator()))
272 return false;
273 }
274
275 unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
276 if (MaxProm == 0)
277 return false;
278
279 unsigned Promoted = 0;
280 for (auto &Cand : LoopToCandidates[&L]) {
281
283 SSAUpdater SSA(&NewPHIs);
284 Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
285
286 // If BFI is set, we will use it to guide the promotions.
287 if (BFI) {
288 auto *BB = Cand.first->getParent();
289 auto InstrCount = BFI->getBlockProfileCount(BB);
290 if (!InstrCount)
291 continue;
292 auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
293 // If the average loop trip count is not greater than 1.5, we skip
294 // promotion.
295 if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
296 continue;
297 }
298
299 PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
300 L.getLoopPreheader(), ExitBlocks,
301 InsertPts, LoopToCandidates, LI);
302 Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
303 Promoted++;
304 if (Promoted >= MaxProm)
305 break;
306
307 (*NumPromoted)++;
308 if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
309 break;
310 }
311
312 LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
313 << L.getLoopDepth() << ")\n");
314 return Promoted != 0;
315 }
316
317private:
318 bool allowSpeculativeCounterPromotion(Loop *LP) {
319 SmallVector<BasicBlock *, 8> ExitingBlocks;
320 L.getExitingBlocks(ExitingBlocks);
321 // Not considierered speculative.
322 if (ExitingBlocks.size() == 1)
323 return true;
324 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
325 return false;
326 return true;
327 }
328
329 // Check whether the loop satisfies the basic conditions needed to perform
330 // Counter Promotions.
331 bool
332 isPromotionPossible(Loop *LP,
333 const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
334 // We can't insert into a catchswitch.
335 if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
336 return isa<CatchSwitchInst>(Exit->getTerminator());
337 }))
338 return false;
339
340 if (!LP->hasDedicatedExits())
341 return false;
342
343 BasicBlock *PH = LP->getLoopPreheader();
344 if (!PH)
345 return false;
346
347 return true;
348 }
349
350 // Returns the max number of Counter Promotions for LP.
351 unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
352 SmallVector<BasicBlock *, 8> LoopExitBlocks;
353 LP->getExitBlocks(LoopExitBlocks);
354 if (!isPromotionPossible(LP, LoopExitBlocks))
355 return 0;
356
357 SmallVector<BasicBlock *, 8> ExitingBlocks;
358 LP->getExitingBlocks(ExitingBlocks);
359
360 // If BFI is set, we do more aggressive promotions based on BFI.
361 if (BFI)
362 return (unsigned)-1;
363
364 // Not considierered speculative.
365 if (ExitingBlocks.size() == 1)
366 return MaxNumOfPromotionsPerLoop;
367
368 if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
369 return 0;
370
371 // Whether the target block is in a loop does not matter:
372 if (SpeculativeCounterPromotionToLoop)
373 return MaxNumOfPromotionsPerLoop;
374
375 // Now check the target block:
376 unsigned MaxProm = MaxNumOfPromotionsPerLoop;
377 for (auto *TargetBlock : LoopExitBlocks) {
378 auto *TargetLoop = LI.getLoopFor(TargetBlock);
379 if (!TargetLoop)
380 continue;
381 unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
382 unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
383 MaxProm =
384 std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
385 PendingCandsInTarget);
386 }
387 return MaxProm;
388 }
389
393 Loop &L;
394 LoopInfo &LI;
396};
397
398enum class ValueProfilingCallType {
399 // Individual values are tracked. Currently used for indiret call target
400 // profiling.
401 Default,
402
403 // MemOp: the memop size value profiling.
404 MemOp
405};
406
407} // end anonymous namespace
408
412 auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
414 };
415 if (!run(M, GetTLI))
416 return PreservedAnalyses::all();
417
419}
420
421bool InstrProfiling::lowerIntrinsics(Function *F) {
422 bool MadeChange = false;
423 PromotionCandidates.clear();
424 for (BasicBlock &BB : *F) {
425 for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
426 if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
427 lowerIncrement(IPIS);
428 MadeChange = true;
429 } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
430 lowerIncrement(IPI);
431 MadeChange = true;
432 } else if (auto *IPC = dyn_cast<InstrProfTimestampInst>(&Instr)) {
433 lowerTimestamp(IPC);
434 MadeChange = true;
435 } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
436 lowerCover(IPC);
437 MadeChange = true;
438 } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
439 lowerValueProfileInst(IPVP);
440 MadeChange = true;
441 } else if (auto *IPMP = dyn_cast<InstrProfMCDCBitmapParameters>(&Instr)) {
442 IPMP->eraseFromParent();
443 MadeChange = true;
444 } else if (auto *IPBU = dyn_cast<InstrProfMCDCTVBitmapUpdate>(&Instr)) {
445 lowerMCDCTestVectorBitmapUpdate(IPBU);
446 MadeChange = true;
447 } else if (auto *IPTU = dyn_cast<InstrProfMCDCCondBitmapUpdate>(&Instr)) {
448 lowerMCDCCondBitmapUpdate(IPTU);
449 MadeChange = true;
450 }
451 }
452 }
453
454 if (!MadeChange)
455 return false;
456
457 promoteCounterLoadStores(F);
458 return true;
459}
460
461bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
462 // Mach-O don't support weak external references.
463 if (TT.isOSBinFormatMachO())
464 return false;
465
466 if (RuntimeCounterRelocation.getNumOccurrences() > 0)
467 return RuntimeCounterRelocation;
468
469 // Fuchsia uses runtime counter relocation by default.
470 return TT.isOSFuchsia();
471}
472
473bool InstrProfiling::isCounterPromotionEnabled() const {
474 if (DoCounterPromotion.getNumOccurrences() > 0)
475 return DoCounterPromotion;
476
477 return Options.DoCounterPromotion;
478}
479
480void InstrProfiling::promoteCounterLoadStores(Function *F) {
481 if (!isCounterPromotionEnabled())
482 return;
483
484 DominatorTree DT(*F);
485 LoopInfo LI(DT);
486 DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
487
488 std::unique_ptr<BlockFrequencyInfo> BFI;
489 if (Options.UseBFIInPromotion) {
490 std::unique_ptr<BranchProbabilityInfo> BPI;
491 BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
492 BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
493 }
494
495 for (const auto &LoadStore : PromotionCandidates) {
496 auto *CounterLoad = LoadStore.first;
497 auto *CounterStore = LoadStore.second;
498 BasicBlock *BB = CounterLoad->getParent();
499 Loop *ParentLoop = LI.getLoopFor(BB);
500 if (!ParentLoop)
501 continue;
502 LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
503 }
504
506
507 // Do a post-order traversal of the loops so that counter updates can be
508 // iteratively hoisted outside the loop nest.
509 for (auto *Loop : llvm::reverse(Loops)) {
510 PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
511 Promoter.run(&TotalCountersPromoted);
512 }
513}
514
516 // On Fuchsia, we only need runtime hook if any counters are present.
517 if (TT.isOSFuchsia())
518 return false;
519
520 return true;
521}
522
523/// Check if the module contains uses of any profiling intrinsics.
525 auto containsIntrinsic = [&](int ID) {
526 if (auto *F = M.getFunction(Intrinsic::getName(ID)))
527 return !F->use_empty();
528 return false;
529 };
530 return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
531 containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
532 containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
533 containsIntrinsic(llvm::Intrinsic::instrprof_timestamp) ||
534 containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
535}
536
538 Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
539 this->M = &M;
540 this->GetTLI = std::move(GetTLI);
541 NamesVar = nullptr;
542 NamesSize = 0;
543 ProfileDataMap.clear();
544 CompilerUsedVars.clear();
545 UsedVars.clear();
546 TT = Triple(M.getTargetTriple());
547
548 bool MadeChange = false;
549 bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
550 if (NeedsRuntimeHook)
551 MadeChange = emitRuntimeHook();
552
553 bool ContainsProfiling = containsProfilingIntrinsics(M);
554 GlobalVariable *CoverageNamesVar =
555 M.getNamedGlobal(getCoverageUnusedNamesVarName());
556 // Improve compile time by avoiding linear scans when there is no work.
557 if (!ContainsProfiling && !CoverageNamesVar)
558 return MadeChange;
559
560 // We did not know how many value sites there would be inside
561 // the instrumented function. This is counting the number of instrumented
562 // target value sites to enter it as field in the profile data variable.
563 for (Function &F : M) {
564 InstrProfCntrInstBase *FirstProfInst = nullptr;
565 for (BasicBlock &BB : F) {
566 for (auto I = BB.begin(), E = BB.end(); I != E; I++) {
567 if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
568 computeNumValueSiteCounts(Ind);
569 else {
570 if (FirstProfInst == nullptr &&
571 (isa<InstrProfIncrementInst>(I) || isa<InstrProfCoverInst>(I)))
572 FirstProfInst = dyn_cast<InstrProfCntrInstBase>(I);
573 // If the MCDCBitmapParameters intrinsic seen, create the bitmaps.
574 if (const auto &Params = dyn_cast<InstrProfMCDCBitmapParameters>(I))
575 static_cast<void>(getOrCreateRegionBitmaps(Params));
576 }
577 }
578 }
579
580 // Use a profile intrinsic to create the region counters and data variable.
581 // Also create the data variable based on the MCDCParams.
582 if (FirstProfInst != nullptr) {
583 static_cast<void>(getOrCreateRegionCounters(FirstProfInst));
584 }
585 }
586
587 for (Function &F : M)
588 MadeChange |= lowerIntrinsics(&F);
589
590 if (CoverageNamesVar) {
591 lowerCoverageData(CoverageNamesVar);
592 MadeChange = true;
593 }
594
595 if (!MadeChange)
596 return false;
597
598 emitVNodes();
599 emitNameData();
600
601 // Emit runtime hook for the cases where the target does not unconditionally
602 // require pulling in profile runtime, and coverage is enabled on code that is
603 // not eliminated by the front-end, e.g. unused functions with internal
604 // linkage.
605 if (!NeedsRuntimeHook && ContainsProfiling)
606 emitRuntimeHook();
607
608 emitRegistration();
609 emitUses();
610 emitInitialization();
611 return true;
612}
613
615 Module &M, const TargetLibraryInfo &TLI,
616 ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
617 LLVMContext &Ctx = M.getContext();
618 auto *ReturnTy = Type::getVoidTy(M.getContext());
619
620 AttributeList AL;
621 if (auto AK = TLI.getExtAttrForI32Param(false))
622 AL = AL.addParamAttribute(M.getContext(), 2, AK);
623
624 assert((CallType == ValueProfilingCallType::Default ||
625 CallType == ValueProfilingCallType::MemOp) &&
626 "Must be Default or MemOp");
627 Type *ParamTypes[] = {
628#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
630 };
631 auto *ValueProfilingCallTy =
632 FunctionType::get(ReturnTy, ArrayRef(ParamTypes), false);
633 StringRef FuncName = CallType == ValueProfilingCallType::Default
636 return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
637}
638
639void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
640 GlobalVariable *Name = Ind->getName();
643 auto &PD = ProfileDataMap[Name];
644 PD.NumValueSites[ValueKind] =
645 std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));
646}
647
648void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
649 // TODO: Value profiling heavily depends on the data section which is omitted
650 // in lightweight mode. We need to move the value profile pointer to the
651 // Counter struct to get this working.
652 assert(
654 "Value profiling is not yet supported with lightweight instrumentation");
655 GlobalVariable *Name = Ind->getName();
656 auto It = ProfileDataMap.find(Name);
657 assert(It != ProfileDataMap.end() && It->second.DataVar &&
658 "value profiling detected in function with no counter incerement");
659
660 GlobalVariable *DataVar = It->second.DataVar;
663 for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
664 Index += It->second.NumValueSites[Kind];
665
666 IRBuilder<> Builder(Ind);
667 bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
668 llvm::InstrProfValueKind::IPVK_MemOPSize);
669 CallInst *Call = nullptr;
670 auto *TLI = &GetTLI(*Ind->getFunction());
671
672 // To support value profiling calls within Windows exception handlers, funclet
673 // information contained within operand bundles needs to be copied over to
674 // the library call. This is required for the IR to be processed by the
675 // WinEHPrepare pass.
677 Ind->getOperandBundlesAsDefs(OpBundles);
678 if (!IsMemOpSize) {
679 Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)};
680 Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
681 OpBundles);
682 } else {
683 Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)};
684 Call = Builder.CreateCall(
685 getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
686 Args, OpBundles);
687 }
688 if (auto AK = TLI->getExtAttrForI32Param(false))
689 Call->addParamAttr(2, AK);
690 Ind->replaceAllUsesWith(Call);
691 Ind->eraseFromParent();
692}
693
694Value *InstrProfiling::getCounterAddress(InstrProfCntrInstBase *I) {
695 auto *Counters = getOrCreateRegionCounters(I);
696 IRBuilder<> Builder(I);
697
698 if (isa<InstrProfTimestampInst>(I))
699 Counters->setAlignment(Align(8));
700
701 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
702 Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());
703
704 if (!isRuntimeCounterRelocationEnabled())
705 return Addr;
706
707 Type *Int64Ty = Type::getInt64Ty(M->getContext());
708 Function *Fn = I->getParent()->getParent();
709 LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
710 if (!BiasLI) {
711 IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
712 auto *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
713 if (!Bias) {
714 // Compiler must define this variable when runtime counter relocation
715 // is being used. Runtime has a weak external reference that is used
716 // to check whether that's the case or not.
717 Bias = new GlobalVariable(
718 *M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
720 Bias->setVisibility(GlobalVariable::HiddenVisibility);
721 // A definition that's weak (linkonce_odr) without being in a COMDAT
722 // section wouldn't lead to link errors, but it would lead to a dead
723 // data word from every TU but one. Putting it in COMDAT ensures there
724 // will be exactly one data slot in the link.
725 if (TT.supportsCOMDAT())
726 Bias->setComdat(M->getOrInsertComdat(Bias->getName()));
727 }
728 BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
729 }
730 auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
731 return Builder.CreateIntToPtr(Add, Addr->getType());
732}
733
734Value *InstrProfiling::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) {
735 auto *Bitmaps = getOrCreateRegionBitmaps(I);
736 IRBuilder<> Builder(I);
737
738 auto *Addr = Builder.CreateConstInBoundsGEP2_32(
739 Bitmaps->getValueType(), Bitmaps, 0, I->getBitmapIndex()->getZExtValue());
740
741 if (isRuntimeCounterRelocationEnabled()) {
742 LLVMContext &Ctx = M->getContext();
744 M->getName().data(),
745 Twine("Runtime counter relocation is presently not supported for MC/DC "
746 "bitmaps."),
747 DS_Warning));
748 }
749
750 return Addr;
751}
752
753void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) {
754 auto *Addr = getCounterAddress(CoverInstruction);
755 IRBuilder<> Builder(CoverInstruction);
756 // We store zero to represent that this block is covered.
757 Builder.CreateStore(Builder.getInt8(0), Addr);
758 CoverInstruction->eraseFromParent();
759}
760
761void InstrProfiling::lowerTimestamp(
762 InstrProfTimestampInst *TimestampInstruction) {
763 assert(TimestampInstruction->getIndex()->isZeroValue() &&
764 "timestamp probes are always the first probe for a function");
765 auto &Ctx = M->getContext();
766 auto *TimestampAddr = getCounterAddress(TimestampInstruction);
767 IRBuilder<> Builder(TimestampInstruction);
768 auto *CalleeTy =
769 FunctionType::get(Type::getVoidTy(Ctx), TimestampAddr->getType(), false);
770 auto Callee = M->getOrInsertFunction(
771 INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy);
772 Builder.CreateCall(Callee, {TimestampAddr});
773 TimestampInstruction->eraseFromParent();
774}
775
776void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
777 auto *Addr = getCounterAddress(Inc);
778
779 IRBuilder<> Builder(Inc);
780 if (Options.Atomic || AtomicCounterUpdateAll ||
781 (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {
782 Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
784 } else {
785 Value *IncStep = Inc->getStep();
786 Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
787 auto *Count = Builder.CreateAdd(Load, Inc->getStep());
788 auto *Store = Builder.CreateStore(Count, Addr);
789 if (isCounterPromotionEnabled())
790 PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
791 }
792 Inc->eraseFromParent();
793}
794
795void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
796 ConstantArray *Names =
797 cast<ConstantArray>(CoverageNamesVar->getInitializer());
798 for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
799 Constant *NC = Names->getOperand(I);
800 Value *V = NC->stripPointerCasts();
801 assert(isa<GlobalVariable>(V) && "Missing reference to function name");
802 GlobalVariable *Name = cast<GlobalVariable>(V);
803
805 ReferencedNames.push_back(Name);
806 if (isa<ConstantExpr>(NC))
807 NC->dropAllReferences();
808 }
809 CoverageNamesVar->eraseFromParent();
810}
811
812void InstrProfiling::lowerMCDCTestVectorBitmapUpdate(
814 IRBuilder<> Builder(Update);
815 auto *Int8Ty = Type::getInt8Ty(M->getContext());
816 auto *Int8PtrTy = PointerType::getUnqual(M->getContext());
817 auto *Int32Ty = Type::getInt32Ty(M->getContext());
818 auto *Int64Ty = Type::getInt64Ty(M->getContext());
819 auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
820 auto *BitmapAddr = getBitmapAddress(Update);
821
822 // Load Temp Val.
823 // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
824 auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp");
825
826 // Calculate byte offset using div8.
827 // %1 = lshr i32 %mcdc.temp, 3
828 auto *BitmapByteOffset = Builder.CreateLShr(Temp, 0x3);
829
830 // Add byte offset to section base byte address.
831 // %2 = zext i32 %1 to i64
832 // %3 = add i64 ptrtoint (ptr @__profbm_test to i64), %2
833 auto *BitmapByteAddr =
834 Builder.CreateAdd(Builder.CreatePtrToInt(BitmapAddr, Int64Ty),
835 Builder.CreateZExtOrBitCast(BitmapByteOffset, Int64Ty));
836
837 // Convert to a pointer.
838 // %4 = inttoptr i32 %3 to ptr
839 BitmapByteAddr = Builder.CreateIntToPtr(BitmapByteAddr, Int8PtrTy);
840
841 // Calculate bit offset into bitmap byte by using div8 remainder (AND ~8)
842 // %5 = and i32 %mcdc.temp, 7
843 // %6 = trunc i32 %5 to i8
844 auto *BitToSet = Builder.CreateTrunc(Builder.CreateAnd(Temp, 0x7), Int8Ty);
845
846 // Shift bit offset left to form a bitmap.
847 // %7 = shl i8 1, %6
848 auto *ShiftedVal = Builder.CreateShl(Builder.getInt8(0x1), BitToSet);
849
850 // Load profile bitmap byte.
851 // %mcdc.bits = load i8, ptr %4, align 1
852 auto *Bitmap = Builder.CreateLoad(Int8Ty, BitmapByteAddr, "mcdc.bits");
853
854 // Perform logical OR of profile bitmap byte and shifted bit offset.
855 // %8 = or i8 %mcdc.bits, %7
856 auto *Result = Builder.CreateOr(Bitmap, ShiftedVal);
857
858 // Store the updated profile bitmap byte.
859 // store i8 %8, ptr %3, align 1
860 Builder.CreateStore(Result, BitmapByteAddr);
861 Update->eraseFromParent();
862}
863
864void InstrProfiling::lowerMCDCCondBitmapUpdate(
866 IRBuilder<> Builder(Update);
867 auto *Int32Ty = Type::getInt32Ty(M->getContext());
868 auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr();
869
870 // Load the MCDC temporary value from the stack.
871 // %mcdc.temp = load i32, ptr %mcdc.addr, align 4
872 auto *Temp = Builder.CreateLoad(Int32Ty, MCDCCondBitmapAddr, "mcdc.temp");
873
874 // Zero-extend the evaluated condition boolean value (0 or 1) by 32bits.
875 // %1 = zext i1 %tobool to i32
876 auto *CondV_32 = Builder.CreateZExt(Update->getCondBool(), Int32Ty);
877
878 // Shift the boolean value left (by the condition's ID) to form a bitmap.
879 // %2 = shl i32 %1, <Update->getCondID()>
880 auto *ShiftedVal = Builder.CreateShl(CondV_32, Update->getCondID());
881
882 // Perform logical OR of the bitmap against the loaded MCDC temporary value.
883 // %3 = or i32 %mcdc.temp, %2
884 auto *Result = Builder.CreateOr(Temp, ShiftedVal);
885
886 // Store the updated temporary value back to the stack.
887 // store i32 %3, ptr %mcdc.addr, align 4
888 Builder.CreateStore(Result, MCDCCondBitmapAddr);
889 Update->eraseFromParent();
890}
891
892/// Get the name of a profiling variable for a particular function.
893static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
894 bool &Renamed) {
896 StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
897 Function *F = Inc->getParent()->getParent();
898 Module *M = F->getParent();
899 if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
901 Renamed = false;
902 return (Prefix + Name).str();
903 }
904 Renamed = true;
905 uint64_t FuncHash = Inc->getHash()->getZExtValue();
906 SmallVector<char, 24> HashPostfix;
907 if (Name.ends_with((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
908 return (Prefix + Name).str();
909 return (Prefix + Name + "." + Twine(FuncHash)).str();
910}
911
913 auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
914 if (!MD)
915 return 0;
916
917 // If the flag is a ConstantAsMetadata, it should be an integer representable
918 // in 64-bits.
919 return cast<ConstantInt>(MD->getValue())->getZExtValue();
920}
921
922static bool enablesValueProfiling(const Module &M) {
923 return isIRPGOFlagSet(&M) ||
924 getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;
925}
926
927// Conservatively returns true if data variables may be referenced by code.
928static bool profDataReferencedByCode(const Module &M) {
929 return enablesValueProfiling(M);
930}
931
932static inline bool shouldRecordFunctionAddr(Function *F) {
933 // Only record function addresses if IR PGO is enabled or if clang value
934 // profiling is enabled. Recording function addresses greatly increases object
935 // file size, because it prevents the inliner from deleting functions that
936 // have been inlined everywhere.
937 if (!profDataReferencedByCode(*F->getParent()))
938 return false;
939
940 // Check the linkage
941 bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
942 if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
943 !HasAvailableExternallyLinkage)
944 return true;
945
946 // A function marked 'alwaysinline' with available_externally linkage can't
947 // have its address taken. Doing so would create an undefined external ref to
948 // the function, which would fail to link.
949 if (HasAvailableExternallyLinkage &&
950 F->hasFnAttribute(Attribute::AlwaysInline))
951 return false;
952
953 // Prohibit function address recording if the function is both internal and
954 // COMDAT. This avoids the profile data variable referencing internal symbols
955 // in COMDAT.
956 if (F->hasLocalLinkage() && F->hasComdat())
957 return false;
958
959 // Check uses of this function for other than direct calls or invokes to it.
960 // Inline virtual functions have linkeOnceODR linkage. When a key method
961 // exists, the vtable will only be emitted in the TU where the key method
962 // is defined. In a TU where vtable is not available, the function won't
963 // be 'addresstaken'. If its address is not recorded here, the profile data
964 // with missing address may be picked by the linker leading to missing
965 // indirect call target info.
966 return F->hasAddressTaken() || F->hasLinkOnceLinkage();
967}
968
969static inline bool shouldUsePublicSymbol(Function *Fn) {
970 // It isn't legal to make an alias of this function at all
971 if (Fn->isDeclarationForLinker())
972 return true;
973
974 // Symbols with local linkage can just use the symbol directly without
975 // introducing relocations
976 if (Fn->hasLocalLinkage())
977 return true;
978
979 // PGO + ThinLTO + CFI cause duplicate symbols to be introduced due to some
980 // unfavorable interaction between the new alias and the alias renaming done
981 // in LowerTypeTests under ThinLTO. For comdat functions that would normally
982 // be deduplicated, but the renaming scheme ends up preventing renaming, since
983 // it creates unique names for each alias, resulting in duplicated symbols. In
984 // the future, we should update the CFI related passes to migrate these
985 // aliases to the same module as the jump-table they refer to will be defined.
986 if (Fn->hasMetadata(LLVMContext::MD_type))
987 return true;
988
989 // For comdat functions, an alias would need the same linkage as the original
990 // function and hidden visibility. There is no point in adding an alias with
991 // identical linkage an visibility to avoid introducing symbolic relocations.
992 if (Fn->hasComdat() &&
994 return true;
995
996 // its OK to use an alias
997 return false;
998}
999
1001 auto *Int8PtrTy = PointerType::getUnqual(Fn->getContext());
1002 // Store a nullptr in __llvm_profd, if we shouldn't use a real address
1003 if (!shouldRecordFunctionAddr(Fn))
1004 return ConstantPointerNull::get(Int8PtrTy);
1005
1006 // If we can't use an alias, we must use the public symbol, even though this
1007 // may require a symbolic relocation.
1008 if (shouldUsePublicSymbol(Fn))
1009 return Fn;
1010
1011 // When possible use a private alias to avoid symbolic relocations.
1013 Fn->getName() + ".local", Fn);
1014
1015 // When the instrumented function is a COMDAT function, we cannot use a
1016 // private alias. If we did, we would create reference to a local label in
1017 // this function's section. If this version of the function isn't selected by
1018 // the linker, then the metadata would introduce a reference to a discarded
1019 // section. So, for COMDAT functions, we need to adjust the linkage of the
1020 // alias. Using hidden visibility avoids a dynamic relocation and an entry in
1021 // the dynamic symbol table.
1022 //
1023 // Note that this handles COMDAT functions with visibility other than Hidden,
1024 // since that case is covered in shouldUsePublicSymbol()
1025 if (Fn->hasComdat()) {
1026 GA->setLinkage(Fn->getLinkage());
1028 }
1029
1030 // appendToCompilerUsed(*Fn->getParent(), {GA});
1031
1032 return GA;
1033}
1034
1036 // Don't do this for Darwin. compiler-rt uses linker magic.
1037 if (TT.isOSDarwin())
1038 return false;
1039 // Use linker script magic to get data/cnts/name start/end.
1040 if (TT.isOSAIX() || TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
1041 TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS() || TT.isOSWindows())
1042 return false;
1043
1044 return true;
1045}
1046
1047void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn,
1048 StringRef VarName) {
1049 bool DataReferencedByCode = profDataReferencedByCode(*M);
1050 bool NeedComdat = needsComdatForCounter(*Fn, *M);
1051 bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
1052
1053 if (!UseComdat)
1054 return;
1055
1056 StringRef GroupName =
1057 TT.isOSBinFormatCOFF() && DataReferencedByCode ? GV->getName() : VarName;
1058 Comdat *C = M->getOrInsertComdat(GroupName);
1059 if (!NeedComdat)
1060 C->setSelectionKind(Comdat::NoDeduplicate);
1061 GV->setComdat(C);
1062 // COFF doesn't allow the comdat group leader to have private linkage, so
1063 // upgrade private linkage to internal linkage to produce a symbol table
1064 // entry.
1065 if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
1067}
1068
1069GlobalVariable *InstrProfiling::setupProfileSection(InstrProfInstBase *Inc,
1070 InstrProfSectKind IPSK) {
1071 GlobalVariable *NamePtr = Inc->getName();
1072
1073 // Match the linkage and visibility of the name global.
1074 Function *Fn = Inc->getParent()->getParent();
1076 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1077
1078 // Use internal rather than private linkage so the counter variable shows up
1079 // in the symbol table when using debug info for correlation.
1081 Linkage == GlobalValue::PrivateLinkage)
1083
1084 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1085 // symbols in the same csect won't be discarded. When there are duplicate weak
1086 // symbols, we can NOT guarantee that the relocations get resolved to the
1087 // intended weak symbol, so we can not ensure the correctness of the relative
1088 // CounterPtr, so we have to use private linkage for counter and data symbols.
1089 if (TT.isOSBinFormatXCOFF()) {
1091 Visibility = GlobalValue::DefaultVisibility;
1092 }
1093 // Move the name variable to the right section. Place them in a COMDAT group
1094 // if the associated function is a COMDAT. This will make sure that only one
1095 // copy of counters of the COMDAT function will be emitted after linking. Keep
1096 // in mind that this pass may run before the inliner, so we need to create a
1097 // new comdat group for the counters and profiling data. If we use the comdat
1098 // of the parent function, that will result in relocations against discarded
1099 // sections.
1100 //
1101 // If the data variable is referenced by code, counters and data have to be
1102 // in different comdats for COFF because the Visual C++ linker will report
1103 // duplicate symbol errors if there are multiple external symbols with the
1104 // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
1105 //
1106 // For ELF, when not using COMDAT, put counters, data and values into a
1107 // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
1108 // allows -z start-stop-gc to discard the entire group when the function is
1109 // discarded.
1110 bool Renamed;
1112 StringRef VarPrefix;
1113 std::string VarName;
1114 if (IPSK == IPSK_cnts) {
1115 VarPrefix = getInstrProfCountersVarPrefix();
1116 VarName = getVarName(Inc, VarPrefix, Renamed);
1117 InstrProfCntrInstBase *CntrIncrement = dyn_cast<InstrProfCntrInstBase>(Inc);
1118 Ptr = createRegionCounters(CntrIncrement, VarName, Linkage);
1119 } else if (IPSK == IPSK_bitmap) {
1120 VarPrefix = getInstrProfBitmapVarPrefix();
1121 VarName = getVarName(Inc, VarPrefix, Renamed);
1122 InstrProfMCDCBitmapInstBase *BitmapUpdate =
1123 dyn_cast<InstrProfMCDCBitmapInstBase>(Inc);
1124 Ptr = createRegionBitmaps(BitmapUpdate, VarName, Linkage);
1125 } else {
1126 llvm_unreachable("Profile Section must be for Counters or Bitmaps");
1127 }
1128
1129 Ptr->setVisibility(Visibility);
1130 // Put the counters and bitmaps in their own sections so linkers can
1131 // remove unneeded sections.
1132 Ptr->setSection(getInstrProfSectionName(IPSK, TT.getObjectFormat()));
1133 Ptr->setLinkage(Linkage);
1134 maybeSetComdat(Ptr, Fn, VarName);
1135 return Ptr;
1136}
1137
1139InstrProfiling::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc,
1141 GlobalValue::LinkageTypes Linkage) {
1142 uint64_t NumBytes = Inc->getNumBitmapBytes()->getZExtValue();
1143 auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M->getContext()), NumBytes);
1144 auto GV = new GlobalVariable(*M, BitmapTy, false, Linkage,
1145 Constant::getNullValue(BitmapTy), Name);
1146 GV->setAlignment(Align(1));
1147 return GV;
1148}
1149
1151InstrProfiling::getOrCreateRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc) {
1152 GlobalVariable *NamePtr = Inc->getName();
1153 auto &PD = ProfileDataMap[NamePtr];
1154 if (PD.RegionBitmaps)
1155 return PD.RegionBitmaps;
1156
1157 // If RegionBitmaps doesn't already exist, create it by first setting up
1158 // the corresponding profile section.
1159 auto *BitmapPtr = setupProfileSection(Inc, IPSK_bitmap);
1160 PD.RegionBitmaps = BitmapPtr;
1161 PD.NumBitmapBytes = Inc->getNumBitmapBytes()->getZExtValue();
1162 return PD.RegionBitmaps;
1163}
1164
1166InstrProfiling::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name,
1167 GlobalValue::LinkageTypes Linkage) {
1168 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1169 auto &Ctx = M->getContext();
1170 GlobalVariable *GV;
1171 if (isa<InstrProfCoverInst>(Inc)) {
1172 auto *CounterTy = Type::getInt8Ty(Ctx);
1173 auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);
1174 // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
1175 std::vector<Constant *> InitialValues(NumCounters,
1176 Constant::getAllOnesValue(CounterTy));
1177 GV = new GlobalVariable(*M, CounterArrTy, false, Linkage,
1178 ConstantArray::get(CounterArrTy, InitialValues),
1179 Name);
1180 GV->setAlignment(Align(1));
1181 } else {
1182 auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
1183 GV = new GlobalVariable(*M, CounterTy, false, Linkage,
1184 Constant::getNullValue(CounterTy), Name);
1185 GV->setAlignment(Align(8));
1186 }
1187 return GV;
1188}
1189
1191InstrProfiling::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) {
1192 GlobalVariable *NamePtr = Inc->getName();
1193 auto &PD = ProfileDataMap[NamePtr];
1194 if (PD.RegionCounters)
1195 return PD.RegionCounters;
1196
1197 // If RegionCounters doesn't already exist, create it by first setting up
1198 // the corresponding profile section.
1199 auto *CounterPtr = setupProfileSection(Inc, IPSK_cnts);
1200 PD.RegionCounters = CounterPtr;
1201
1202 if (DebugInfoCorrelate) {
1203 LLVMContext &Ctx = M->getContext();
1204 Function *Fn = Inc->getParent()->getParent();
1205 if (auto *SP = Fn->getSubprogram()) {
1206 DIBuilder DB(*M, true, SP->getUnit());
1207 Metadata *FunctionNameAnnotation[] = {
1210 };
1211 Metadata *CFGHashAnnotation[] = {
1214 };
1215 Metadata *NumCountersAnnotation[] = {
1218 };
1219 auto Annotations = DB.getOrCreateArray({
1220 MDNode::get(Ctx, FunctionNameAnnotation),
1221 MDNode::get(Ctx, CFGHashAnnotation),
1222 MDNode::get(Ctx, NumCountersAnnotation),
1223 });
1224 auto *DICounter = DB.createGlobalVariableExpression(
1225 SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
1226 /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),
1227 CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
1228 /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
1229 Annotations);
1230 CounterPtr->addDebugInfo(DICounter);
1231 DB.finalize();
1232 }
1233
1234 // Mark the counter variable as used so that it isn't optimized out.
1235 CompilerUsedVars.push_back(PD.RegionCounters);
1236 }
1237
1238 // Create the data variable (if it doesn't already exist).
1239 createDataVariable(Inc);
1240
1241 return PD.RegionCounters;
1242}
1243
1244void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) {
1245 // When debug information is correlated to profile data, a data variable
1246 // is not needed.
1248 return;
1249
1250 GlobalVariable *NamePtr = Inc->getName();
1251 auto &PD = ProfileDataMap[NamePtr];
1252
1253 // Return if data variable was already created.
1254 if (PD.DataVar)
1255 return;
1256
1257 LLVMContext &Ctx = M->getContext();
1258
1259 Function *Fn = Inc->getParent()->getParent();
1261 GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
1262
1263 // Due to the limitation of binder as of 2021/09/28, the duplicate weak
1264 // symbols in the same csect won't be discarded. When there are duplicate weak
1265 // symbols, we can NOT guarantee that the relocations get resolved to the
1266 // intended weak symbol, so we can not ensure the correctness of the relative
1267 // CounterPtr, so we have to use private linkage for counter and data symbols.
1268 if (TT.isOSBinFormatXCOFF()) {
1270 Visibility = GlobalValue::DefaultVisibility;
1271 }
1272
1273 bool DataReferencedByCode = profDataReferencedByCode(*M);
1274 bool NeedComdat = needsComdatForCounter(*Fn, *M);
1275 bool Renamed;
1276
1277 // The Data Variable section is anchored to profile counters.
1278 std::string CntsVarName =
1280 std::string DataVarName =
1281 getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
1282
1283 auto *Int8PtrTy = PointerType::getUnqual(Ctx);
1284 // Allocate statically the array of pointers to value profile nodes for
1285 // the current function.
1286 Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
1287 uint64_t NS = 0;
1288 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1289 NS += PD.NumValueSites[Kind];
1290 if (NS > 0 && ValueProfileStaticAlloc &&
1292 ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
1293 auto *ValuesVar = new GlobalVariable(
1294 *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
1295 getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
1296 ValuesVar->setVisibility(Visibility);
1297 ValuesVar->setSection(
1298 getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
1299 ValuesVar->setAlignment(Align(8));
1300 maybeSetComdat(ValuesVar, Fn, CntsVarName);
1301 ValuesPtrExpr = ValuesVar;
1302 }
1303
1304 uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
1305 auto *CounterPtr = PD.RegionCounters;
1306
1307 uint64_t NumBitmapBytes = PD.NumBitmapBytes;
1308
1309 // Create data variable.
1310 auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext());
1311 auto *Int16Ty = Type::getInt16Ty(Ctx);
1312 auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
1313 Type *DataTypes[] = {
1314#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1316 };
1317 auto *DataTy = StructType::get(Ctx, ArrayRef(DataTypes));
1318
1319 Constant *FunctionAddr = getFuncAddrForProfData(Fn);
1320
1321 Constant *Int16ArrayVals[IPVK_Last + 1];
1322 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1323 Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
1324
1325 // If the data variable is not referenced by code (if we don't emit
1326 // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
1327 // data variable live under linker GC, the data variable can be private. This
1328 // optimization applies to ELF.
1329 //
1330 // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
1331 // to be false.
1332 //
1333 // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
1334 // that other copies must have the same CFG and cannot have value profiling.
1335 // If no hash suffix, other profd copies may be referenced by code.
1336 if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1337 (TT.isOSBinFormatELF() ||
1338 (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
1340 Visibility = GlobalValue::DefaultVisibility;
1341 }
1342 auto *Data =
1343 new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName);
1344 // Reference the counter variable with a label difference (link-time
1345 // constant).
1346 auto *RelativeCounterPtr =
1347 ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
1348 ConstantExpr::getPtrToInt(Data, IntPtrTy));
1349
1350 // Bitmaps are relative to the same data variable as profile counters.
1351 GlobalVariable *BitmapPtr = PD.RegionBitmaps;
1352 Constant *RelativeBitmapPtr = ConstantInt::get(IntPtrTy, 0);
1353
1354 if (BitmapPtr != nullptr) {
1355 RelativeBitmapPtr =
1357 ConstantExpr::getPtrToInt(Data, IntPtrTy));
1358 }
1359
1360 Constant *DataVals[] = {
1361#define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1363 };
1364 Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
1365
1366 Data->setVisibility(Visibility);
1367 Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
1368 Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
1369 maybeSetComdat(Data, Fn, CntsVarName);
1370
1371 PD.DataVar = Data;
1372
1373 // Mark the data variable as used so that it isn't stripped out.
1374 CompilerUsedVars.push_back(Data);
1375 // Now that the linkage set by the FE has been passed to the data and counter
1376 // variables, reset Name variable's linkage and visibility to private so that
1377 // it can be removed later by the compiler.
1379 // Collect the referenced names to be used by emitNameData.
1380 ReferencedNames.push_back(NamePtr);
1381}
1382
1383void InstrProfiling::emitVNodes() {
1384 if (!ValueProfileStaticAlloc)
1385 return;
1386
1387 // For now only support this on platforms that do
1388 // not require runtime registration to discover
1389 // named section start/end.
1391 return;
1392
1393 size_t TotalNS = 0;
1394 for (auto &PD : ProfileDataMap) {
1395 for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1396 TotalNS += PD.second.NumValueSites[Kind];
1397 }
1398
1399 if (!TotalNS)
1400 return;
1401
1402 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1403// Heuristic for small programs with very few total value sites.
1404// The default value of vp-counters-per-site is chosen based on
1405// the observation that large apps usually have a low percentage
1406// of value sites that actually have any profile data, and thus
1407// the average number of counters per site is low. For small
1408// apps with very few sites, this may not be true. Bump up the
1409// number of counters in this case.
1410#define INSTR_PROF_MIN_VAL_COUNTS 10
1411 if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
1412 NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
1413
1414 auto &Ctx = M->getContext();
1415 Type *VNodeTypes[] = {
1416#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1418 };
1419 auto *VNodeTy = StructType::get(Ctx, ArrayRef(VNodeTypes));
1420
1421 ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
1422 auto *VNodesVar = new GlobalVariable(
1423 *M, VNodesTy, false, GlobalValue::PrivateLinkage,
1425 VNodesVar->setSection(
1426 getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
1427 VNodesVar->setAlignment(M->getDataLayout().getABITypeAlign(VNodesTy));
1428 // VNodesVar is used by runtime but not referenced via relocation by other
1429 // sections. Conservatively make it linker retained.
1430 UsedVars.push_back(VNodesVar);
1431}
1432
1433void InstrProfiling::emitNameData() {
1434 std::string UncompressedData;
1435
1436 if (ReferencedNames.empty())
1437 return;
1438
1439 std::string CompressedNameStr;
1440 if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
1442 report_fatal_error(Twine(toString(std::move(E))), false);
1443 }
1444
1445 auto &Ctx = M->getContext();
1446 auto *NamesVal =
1447 ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false);
1448 NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
1451 NamesSize = CompressedNameStr.size();
1452 NamesVar->setSection(
1453 getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
1454 // On COFF, it's important to reduce the alignment down to 1 to prevent the
1455 // linker from inserting padding before the start of the names section or
1456 // between names entries.
1457 NamesVar->setAlignment(Align(1));
1458 // NamesVar is used by runtime but not referenced via relocation by other
1459 // sections. Conservatively make it linker retained.
1460 UsedVars.push_back(NamesVar);
1461
1462 for (auto *NamePtr : ReferencedNames)
1463 NamePtr->eraseFromParent();
1464}
1465
1466void InstrProfiling::emitRegistration() {
1468 return;
1469
1470 // Construct the function.
1471 auto *VoidTy = Type::getVoidTy(M->getContext());
1472 auto *VoidPtrTy = PointerType::getUnqual(M->getContext());
1473 auto *Int64Ty = Type::getInt64Ty(M->getContext());
1474 auto *RegisterFTy = FunctionType::get(VoidTy, false);
1475 auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1477 RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1478 if (Options.NoRedZone)
1479 RegisterF->addFnAttr(Attribute::NoRedZone);
1480
1481 auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1482 auto *RuntimeRegisterF =
1485
1486 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
1487 for (Value *Data : CompilerUsedVars)
1488 if (!isa<Function>(Data))
1489 IRB.CreateCall(RuntimeRegisterF, Data);
1490 for (Value *Data : UsedVars)
1491 if (Data != NamesVar && !isa<Function>(Data))
1492 IRB.CreateCall(RuntimeRegisterF, Data);
1493
1494 if (NamesVar) {
1495 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1496 auto *NamesRegisterTy =
1497 FunctionType::get(VoidTy, ArrayRef(ParamTypes), false);
1498 auto *NamesRegisterF =
1501 IRB.CreateCall(NamesRegisterF, {NamesVar, IRB.getInt64(NamesSize)});
1502 }
1503
1504 IRB.CreateRetVoid();
1505}
1506
1507bool InstrProfiling::emitRuntimeHook() {
1508 // We expect the linker to be invoked with -u<hook_var> flag for Linux
1509 // in which case there is no need to emit the external variable.
1510 if (TT.isOSLinux() || TT.isOSAIX())
1511 return false;
1512
1513 // If the module's provided its own runtime, we don't need to do anything.
1514 if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
1515 return false;
1516
1517 // Declare an external variable that will pull in the runtime initialization.
1518 auto *Int32Ty = Type::getInt32Ty(M->getContext());
1519 auto *Var =
1522 Var->setVisibility(GlobalValue::HiddenVisibility);
1523
1524 if (TT.isOSBinFormatELF() && !TT.isPS()) {
1525 // Mark the user variable as used so that it isn't stripped out.
1526 CompilerUsedVars.push_back(Var);
1527 } else {
1528 // Make a function that uses it.
1532 User->addFnAttr(Attribute::NoInline);
1533 if (Options.NoRedZone)
1534 User->addFnAttr(Attribute::NoRedZone);
1535 User->setVisibility(GlobalValue::HiddenVisibility);
1536 if (TT.supportsCOMDAT())
1537 User->setComdat(M->getOrInsertComdat(User->getName()));
1538
1539 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
1540 auto *Load = IRB.CreateLoad(Int32Ty, Var);
1541 IRB.CreateRet(Load);
1542
1543 // Mark the function as used so that it isn't stripped out.
1544 CompilerUsedVars.push_back(User);
1545 }
1546 return true;
1547}
1548
1549void InstrProfiling::emitUses() {
1550 // The metadata sections are parallel arrays. Optimizers (e.g.
1551 // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
1552 // we conservatively retain all unconditionally in the compiler.
1553 //
1554 // On ELF and Mach-O, the linker can guarantee the associated sections will be
1555 // retained or discarded as a unit, so llvm.compiler.used is sufficient.
1556 // Similarly on COFF, if prof data is not referenced by code we use one comdat
1557 // and ensure this GC property as well. Otherwise, we have to conservatively
1558 // make all of the sections retained by the linker.
1559 if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
1561 appendToCompilerUsed(*M, CompilerUsedVars);
1562 else
1563 appendToUsed(*M, CompilerUsedVars);
1564
1565 // We do not add proper references from used metadata sections to NamesVar and
1566 // VNodesVar, so we have to be conservative and place them in llvm.used
1567 // regardless of the target,
1568 appendToUsed(*M, UsedVars);
1569}
1570
1571void InstrProfiling::emitInitialization() {
1572 // Create ProfileFileName variable. Don't don't this for the
1573 // context-sensitive instrumentation lowering: This lowering is after
1574 // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1575 // have already create the variable before LTO/ThinLTO linking.
1576 if (!IsCS)
1578 Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1579 if (!RegisterF)
1580 return;
1581
1582 // Create the initialization function.
1583 auto *VoidTy = Type::getVoidTy(M->getContext());
1584 auto *F = Function::Create(FunctionType::get(VoidTy, false),
1587 F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1588 F->addFnAttr(Attribute::NoInline);
1589 if (Options.NoRedZone)
1590 F->addFnAttr(Attribute::NoRedZone);
1591
1592 // Add the basic block and the necessary calls.
1593 IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1594 IRB.CreateCall(RegisterF, {});
1595 IRB.CreateRetVoid();
1596
1597 appendToGlobalCtors(*M, F, 0);
1598}
This file contains the simple types necessary to represent the attributes associated with functions a...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
static unsigned InstrCount
#define LLVM_DEBUG(X)
Definition: Debug.h:101
@ Default
Definition: DwarfDebug.cpp:87
uint64_t Addr
std::string Name
Hexagon Hardware Loops
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static bool enablesValueProfiling(const Module &M)
static bool shouldRecordFunctionAddr(Function *F)
static bool needsRuntimeHookUnconditionally(const Triple &TT)
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
#define INSTR_PROF_MIN_VAL_COUNTS
static Constant * getFuncAddrForProfData(Function *Fn)
static bool shouldUsePublicSymbol(Function *Fn)
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag)
static bool profDataReferencedByCode(const Module &M)
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
Memory SSA
Definition: MemorySSA.cpp:71
Module.h This file contains the declarations for the Module class.
IntegerType * Int32Ty
This file provides the interface for IR based instrumentation passes ( (profile-gen,...
if(VerifyEach)
FunctionAnalysisManager FAM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file contains some templates that are useful if you are working with the STL at all.
This file defines the SmallVector class.
A container for analyses that lazily runs them and caches their results.
Definition: PassManager.h:649
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:803
Annotations lets you mark points and ranges inside source code, for tests:
Definition: Annotations.h:53
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:654
@ Add
*p = old + v
Definition: Instructions.h:742
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
iterator end()
Definition: BasicBlock.h:450
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:437
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:446
const Instruction & front() const
Definition: BasicBlock.h:460
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:206
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:213
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:228
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Analysis providing branch probability information.
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
This class represents a function call, abstracting a target machine's calling convention.
@ NoDeduplicate
No deduplication is performed.
Definition: Comdat.h:39
ConstantArray - Constant Array Declarations.
Definition: Constants.h:409
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1235
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:506
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2793
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2460
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2028
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:888
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:146
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1691
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1300
This is an important base class in LLVM.
Definition: Constant.h:41
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:403
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:356
bool isZeroValue() const
Return true if the value is negative zero or null value.
Definition: Constants.cpp:62
Diagnostic information for the PGO profiler.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
Lightweight error class with error context and mandatory checking.
Definition: Error.h:160
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:168
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:162
const BasicBlock & getEntryBlock() const
Definition: Function.h:778
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1781
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition: Function.cpp:341
static GlobalAlias * create(Type *Ty, unsigned AddressSpace, LinkageTypes Linkage, const Twine &Name, Constant *Aliasee, Module *Parent)
If a parent module is specified, the alias is automatically inserted into the end of the specified mo...
Definition: Globals.cpp:517
bool hasMetadata() const
Return true if this value has any metadata attached to it.
Definition: Value.h:589
void setAlignment(Align Align)
Sets the alignment attribute of the GlobalObject.
Definition: Globals.cpp:128
void setComdat(Comdat *C)
Definition: Globals.cpp:196
bool hasComdat() const
Definition: GlobalObject.h:128
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:250
VisibilityTypes getVisibility() const
Definition: GlobalValue.h:244
LinkageTypes getLinkage() const
Definition: GlobalValue.h:541
bool hasLocalLinkage() const
Definition: GlobalValue.h:523
bool hasPrivateLinkage() const
Definition: GlobalValue.h:522
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:532
bool isDeclarationForLinker() const
Definition: GlobalValue.h:614
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:62
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:63
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:454
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2639
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:962
A base class for all instrprof counter intrinsics.
ConstantInt * getIndex() const
ConstantInt * getNumCounters() const
static const char * FunctionNameAttributeName
static const char * CFGHashAttributeName
static const char * NumCountersAttributeName
This represents the llvm.instrprof.cover intrinsic.
This represents the llvm.instrprof.increment intrinsic.
A base class for all instrprof intrinsics.
GlobalVariable * getName() const
ConstantInt * getHash() const
A base class for instrprof mcdc intrinsics that require global bitmap bytes.
ConstantInt * getNumBitmapBytes() const
This represents the llvm.instrprof.mcdc.condbitmap.update intrinsic.
This represents the llvm.instrprof.mcdc.tvbitmap.update intrinsic.
This represents the llvm.instrprof.timestamp intrinsic.
This represents the llvm.instrprof.value.profile intrinsic.
ConstantInt * getIndex() const
ConstantInt * getValueKind() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
const BasicBlock * getParent() const
Definition: Instruction.h:139
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:93
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:75
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Helper class for promoting a collection of loads and stores into SSA Form using the SSAUpdater.
Definition: SSAUpdater.h:150
virtual void doExtraRewritesBeforeFinalDeletion()
This hook is invoked after all the stores are found and inserted as available values.
Definition: SSAUpdater.h:175
An instruction for reading from memory.
Definition: Instructions.h:177
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:44
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1504
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:564
Root of the metadata hierarchy.
Definition: Metadata.h:62
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Definition: DerivedTypes.h:662
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:172
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:175
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:178
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition: SSAUpdater.h:40
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:366
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:451
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:575
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:380
Analysis pass providing the TargetLibraryInfo.
Provides information about what library functions are available for the current target.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
Definition: Triple.h:383
bool isOSBinFormatMachO() const
Tests whether the environment is MachO.
Definition: Triple.h:691
bool supportsCOMDAT() const
Tests whether the target supports comdat.
Definition: Triple.h:1005
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
Definition: Triple.h:683
bool isOSBinFormatXCOFF() const
Tests whether the OS uses the XCOFF binary format.
Definition: Triple.h:701
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:637
bool isOSAIX() const
Tests whether the OS is AIX.
Definition: Triple.h:669
bool isPS() const
Tests whether the target is the PS4 or PS5 platform.
Definition: Triple.h:725
bool isOSFuchsia() const
Definition: Triple.h:547
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Definition: Triple.h:678
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt16Ty(LLVMContext &C)
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
Definition: Type.h:129
static IntegerType * getInt8Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
Value * getOperand(unsigned i) const
Definition: User.h:169
unsigned getNumOperands() const
Definition: User.h:191
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
ValueKind
Value kinds.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:1010
@ PD
PD - Prefix code for packed double precision vector floating point operations performed in the SSE re...
Definition: X86BaseInfo.h:727
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
StringRef toStringRef(const std::optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:90
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:157
StringRef getInstrProfBitmapVarPrefix()
Return the name prefix of profile bitmap variables.
Definition: InstrProf.h:99
cl::opt< bool > DoInstrProfNameCompression
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:665
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:93
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:124
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:224
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:152
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:102
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1733
StringRef getInstrProfCounterBiasVarName()
Definition: InstrProf.h:167
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:428
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition: InstrProf.h:163
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:133
Error collectPGOFuncNameStrings(ArrayRef< GlobalVariable * > NameVars, std::string &Result, bool doCompression=true)
Produce Result string with the same format described above.
Definition: InstrProf.cpp:612
InstrProfSectKind
Definition: InstrProf.h:58
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:96
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1740
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:605
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition: InstrProf.h:85
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:144
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
@ Add
Sum of integers.
cl::opt< bool > DebugInfoCorrelate
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1274
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1322
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1345
@ DS_Warning
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:73
bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src, const BasicBlock &Dest)
auto predecessors(const MachineBasicBlock *BB)
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:79
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:139
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition: InstrProf.h:109
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1300
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:105
#define NC
Definition: regutils.h:42
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
std::string InstrProfileOutput
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117