LLVM  15.0.0git
InstrProfiling.cpp
Go to the documentation of this file.
1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10 // It also builds the data structures and initialization code needed for
11 // updating execution counts and emitting the profile at runtime.
12 //
13 //===----------------------------------------------------------------------===//
14 
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/ADT/Twine.h"
23 #include "llvm/Analysis/LoopInfo.h"
25 #include "llvm/IR/Attributes.h"
26 #include "llvm/IR/BasicBlock.h"
27 #include "llvm/IR/Constant.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DIBuilder.h"
30 #include "llvm/IR/DerivedTypes.h"
31 #include "llvm/IR/DiagnosticInfo.h"
32 #include "llvm/IR/Dominators.h"
33 #include "llvm/IR/Function.h"
34 #include "llvm/IR/GlobalValue.h"
35 #include "llvm/IR/GlobalVariable.h"
36 #include "llvm/IR/IRBuilder.h"
37 #include "llvm/IR/Instruction.h"
38 #include "llvm/IR/Instructions.h"
39 #include "llvm/IR/IntrinsicInst.h"
40 #include "llvm/IR/Module.h"
41 #include "llvm/IR/Type.h"
42 #include "llvm/InitializePasses.h"
43 #include "llvm/Pass.h"
46 #include "llvm/Support/Casting.h"
48 #include "llvm/Support/Error.h"
52 #include <algorithm>
53 #include <cassert>
54 #include <cstdint>
55 #include <string>
56 
57 using namespace llvm;
58 
59 #define DEBUG_TYPE "instrprof"
60 
61 namespace llvm {
63  DebugInfoCorrelate("debug-info-correlate",
64  cl::desc("Use debug info to correlate profiles."),
65  cl::init(false));
66 } // namespace llvm
67 
68 namespace {
69 
70 cl::opt<bool> DoHashBasedCounterSplit(
71  "hash-based-counter-split",
72  cl::desc("Rename counter variable of a comdat function based on cfg hash"),
73  cl::init(true));
74 
76  RuntimeCounterRelocation("runtime-counter-relocation",
77  cl::desc("Enable relocating counters at runtime."),
78  cl::init(false));
79 
80 cl::opt<bool> ValueProfileStaticAlloc(
81  "vp-static-alloc",
82  cl::desc("Do static counter allocation for value profiler"),
83  cl::init(true));
84 
85 cl::opt<double> NumCountersPerValueSite(
86  "vp-counters-per-site",
87  cl::desc("The average number of profile counters allocated "
88  "per value profiling site."),
89  // This is set to a very small value because in real programs, only
90  // a very small percentage of value sites have non-zero targets, e.g, 1/30.
91  // For those sites with non-zero profile, the average number of targets
92  // is usually smaller than 2.
93  cl::init(1.0));
94 
95 cl::opt<bool> AtomicCounterUpdateAll(
96  "instrprof-atomic-counter-update-all",
97  cl::desc("Make all profile counter updates atomic (for testing only)"),
98  cl::init(false));
99 
100 cl::opt<bool> AtomicCounterUpdatePromoted(
101  "atomic-counter-update-promoted",
102  cl::desc("Do counter update using atomic fetch add "
103  " for promoted counters only"),
104  cl::init(false));
105 
106 cl::opt<bool> AtomicFirstCounter(
107  "atomic-first-counter",
108  cl::desc("Use atomic fetch add for first counter in a function (usually "
109  "the entry counter)"),
110  cl::init(false));
111 
112 // If the option is not specified, the default behavior about whether
113 // counter promotion is done depends on how instrumentaiton lowering
114 // pipeline is setup, i.e., the default value of true of this option
115 // does not mean the promotion will be done by default. Explicitly
116 // setting this option can override the default behavior.
117 cl::opt<bool> DoCounterPromotion("do-counter-promotion",
118  cl::desc("Do counter register promotion"),
119  cl::init(false));
120 cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
121  "max-counter-promotions-per-loop", cl::init(20),
122  cl::desc("Max number counter promotions per loop to avoid"
123  " increasing register pressure too much"));
124 
125 // A debug option
127  MaxNumOfPromotions("max-counter-promotions", cl::init(-1),
128  cl::desc("Max number of allowed counter promotions"));
129 
130 cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
131  "speculative-counter-promotion-max-exiting", cl::init(3),
132  cl::desc("The max number of exiting blocks of a loop to allow "
133  " speculative counter promotion"));
134 
135 cl::opt<bool> SpeculativeCounterPromotionToLoop(
136  "speculative-counter-promotion-to-loop",
137  cl::desc("When the option is false, if the target block is in a loop, "
138  "the promotion will be disallowed unless the promoted counter "
139  " update can be further/iteratively promoted into an acyclic "
140  " region."));
141 
142 cl::opt<bool> IterativeCounterPromotion(
143  "iterative-counter-promotion", cl::init(true),
144  cl::desc("Allow counter promotion across the whole loop nest."));
145 
146 cl::opt<bool> SkipRetExitBlock(
147  "skip-ret-exit-block", cl::init(true),
148  cl::desc("Suppress counter promotion if exit blocks contain ret."));
149 
150 class InstrProfilingLegacyPass : public ModulePass {
151  InstrProfiling InstrProf;
152 
153 public:
154  static char ID;
155 
156  InstrProfilingLegacyPass() : ModulePass(ID) {}
157  InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false)
158  : ModulePass(ID), InstrProf(Options, IsCS) {
160  }
161 
162  StringRef getPassName() const override {
163  return "Frontend instrumentation-based coverage lowering";
164  }
165 
166  bool runOnModule(Module &M) override {
167  auto GetTLI = [this](Function &F) -> TargetLibraryInfo & {
168  return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F);
169  };
170  return InstrProf.run(M, GetTLI);
171  }
172 
173  void getAnalysisUsage(AnalysisUsage &AU) const override {
174  AU.setPreservesCFG();
176  }
177 };
178 
179 ///
180 /// A helper class to promote one counter RMW operation in the loop
181 /// into register update.
182 ///
183 /// RWM update for the counter will be sinked out of the loop after
184 /// the transformation.
185 ///
186 class PGOCounterPromoterHelper : public LoadAndStorePromoter {
187 public:
188  PGOCounterPromoterHelper(
190  BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
191  ArrayRef<Instruction *> InsertPts,
193  LoopInfo &LI)
194  : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
195  InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
196  assert(isa<LoadInst>(L));
197  assert(isa<StoreInst>(S));
198  SSA.AddAvailableValue(PH, Init);
199  }
200 
201  void doExtraRewritesBeforeFinalDeletion() override {
202  for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
203  BasicBlock *ExitBlock = ExitBlocks[i];
204  Instruction *InsertPos = InsertPts[i];
205  // Get LiveIn value into the ExitBlock. If there are multiple
206  // predecessors, the value is defined by a PHI node in this
207  // block.
208  Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
209  Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
210  Type *Ty = LiveInValue->getType();
211  IRBuilder<> Builder(InsertPos);
212  if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Addr)) {
213  // If isRuntimeCounterRelocationEnabled() is true then the address of
214  // the store instruction is computed with two instructions in
215  // InstrProfiling::getCounterAddress(). We need to copy those
216  // instructions to this block to compute Addr correctly.
217  // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
218  // %Addr = inttoptr i64 %BiasAdd to i64*
219  auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
220  assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
221  Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
222  Addr = Builder.CreateIntToPtr(BiasInst, Ty->getPointerTo());
223  }
224  if (AtomicCounterUpdatePromoted)
225  // automic update currently can only be promoted across the current
226  // loop, not the whole loop nest.
227  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
228  MaybeAlign(),
230  else {
231  LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
232  auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
233  auto *NewStore = Builder.CreateStore(NewVal, Addr);
234 
235  // Now update the parent loop's candidate list:
236  if (IterativeCounterPromotion) {
237  auto *TargetLoop = LI.getLoopFor(ExitBlock);
238  if (TargetLoop)
239  LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
240  }
241  }
242  }
243  }
244 
245 private:
247  ArrayRef<BasicBlock *> ExitBlocks;
248  ArrayRef<Instruction *> InsertPts;
250  LoopInfo &LI;
251 };
252 
253 /// A helper class to do register promotion for all profile counter
254 /// updates in a loop.
255 ///
256 class PGOCounterPromoter {
257 public:
258  PGOCounterPromoter(
260  Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
261  : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {
262 
263  // Skip collection of ExitBlocks and InsertPts for loops that will not be
264  // able to have counters promoted.
265  SmallVector<BasicBlock *, 8> LoopExitBlocks;
267 
268  L.getExitBlocks(LoopExitBlocks);
269  if (!isPromotionPossible(&L, LoopExitBlocks))
270  return;
271 
272  for (BasicBlock *ExitBlock : LoopExitBlocks) {
273  if (BlockSet.insert(ExitBlock).second) {
274  ExitBlocks.push_back(ExitBlock);
275  InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
276  }
277  }
278  }
279 
280  bool run(int64_t *NumPromoted) {
281  // Skip 'infinite' loops:
282  if (ExitBlocks.size() == 0)
283  return false;
284 
285  // Skip if any of the ExitBlocks contains a ret instruction.
286  // This is to prevent dumping of incomplete profile -- if the
287  // the loop is a long running loop and dump is called in the middle
288  // of the loop, the result profile is incomplete.
289  // FIXME: add other heuristics to detect long running loops.
290  if (SkipRetExitBlock) {
291  for (auto BB : ExitBlocks)
292  if (isa<ReturnInst>(BB->getTerminator()))
293  return false;
294  }
295 
296  unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
297  if (MaxProm == 0)
298  return false;
299 
300  unsigned Promoted = 0;
301  for (auto &Cand : LoopToCandidates[&L]) {
302 
304  SSAUpdater SSA(&NewPHIs);
305  Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
306 
307  // If BFI is set, we will use it to guide the promotions.
308  if (BFI) {
309  auto *BB = Cand.first->getParent();
310  auto InstrCount = BFI->getBlockProfileCount(BB);
311  if (!InstrCount)
312  continue;
313  auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
314  // If the average loop trip count is not greater than 1.5, we skip
315  // promotion.
316  if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
317  continue;
318  }
319 
320  PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
321  L.getLoopPreheader(), ExitBlocks,
322  InsertPts, LoopToCandidates, LI);
323  Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
324  Promoted++;
325  if (Promoted >= MaxProm)
326  break;
327 
328  (*NumPromoted)++;
329  if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
330  break;
331  }
332 
333  LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
334  << L.getLoopDepth() << ")\n");
335  return Promoted != 0;
336  }
337 
338 private:
339  bool allowSpeculativeCounterPromotion(Loop *LP) {
340  SmallVector<BasicBlock *, 8> ExitingBlocks;
341  L.getExitingBlocks(ExitingBlocks);
342  // Not considierered speculative.
343  if (ExitingBlocks.size() == 1)
344  return true;
345  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
346  return false;
347  return true;
348  }
349 
350  // Check whether the loop satisfies the basic conditions needed to perform
351  // Counter Promotions.
352  bool
353  isPromotionPossible(Loop *LP,
354  const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
355  // We can't insert into a catchswitch.
356  if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
357  return isa<CatchSwitchInst>(Exit->getTerminator());
358  }))
359  return false;
360 
361  if (!LP->hasDedicatedExits())
362  return false;
363 
364  BasicBlock *PH = LP->getLoopPreheader();
365  if (!PH)
366  return false;
367 
368  return true;
369  }
370 
371  // Returns the max number of Counter Promotions for LP.
372  unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
373  SmallVector<BasicBlock *, 8> LoopExitBlocks;
374  LP->getExitBlocks(LoopExitBlocks);
375  if (!isPromotionPossible(LP, LoopExitBlocks))
376  return 0;
377 
378  SmallVector<BasicBlock *, 8> ExitingBlocks;
379  LP->getExitingBlocks(ExitingBlocks);
380 
381  // If BFI is set, we do more aggressive promotions based on BFI.
382  if (BFI)
383  return (unsigned)-1;
384 
385  // Not considierered speculative.
386  if (ExitingBlocks.size() == 1)
387  return MaxNumOfPromotionsPerLoop;
388 
389  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
390  return 0;
391 
392  // Whether the target block is in a loop does not matter:
393  if (SpeculativeCounterPromotionToLoop)
394  return MaxNumOfPromotionsPerLoop;
395 
396  // Now check the target block:
397  unsigned MaxProm = MaxNumOfPromotionsPerLoop;
398  for (auto *TargetBlock : LoopExitBlocks) {
399  auto *TargetLoop = LI.getLoopFor(TargetBlock);
400  if (!TargetLoop)
401  continue;
402  unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
403  unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
404  MaxProm =
405  std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
406  PendingCandsInTarget);
407  }
408  return MaxProm;
409  }
410 
412  SmallVector<BasicBlock *, 8> ExitBlocks;
414  Loop &L;
415  LoopInfo &LI;
417 };
418 
419 enum class ValueProfilingCallType {
420  // Individual values are tracked. Currently used for indiret call target
421  // profiling.
422  Default,
423 
424  // MemOp: the memop size value profiling.
425  MemOp
426 };
427 
428 } // end anonymous namespace
429 
433  auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
435  };
436  if (!run(M, GetTLI))
437  return PreservedAnalyses::all();
438 
439  return PreservedAnalyses::none();
440 }
441 
443 INITIALIZE_PASS_BEGIN(InstrProfilingLegacyPass, "instrprof",
444  "Frontend instrumentation-based coverage lowering.",
445  false, false)
447 INITIALIZE_PASS_END(InstrProfilingLegacyPass, "instrprof",
448  "Frontend instrumentation-based coverage lowering.", false,
449  false)
450 
451 ModulePass *
453  bool IsCS) {
454  return new InstrProfilingLegacyPass(Options, IsCS);
455 }
456 
457 bool InstrProfiling::lowerIntrinsics(Function *F) {
458  bool MadeChange = false;
459  PromotionCandidates.clear();
460  for (BasicBlock &BB : *F) {
461  for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
462  if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
463  lowerIncrement(IPIS);
464  MadeChange = true;
465  } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
466  lowerIncrement(IPI);
467  MadeChange = true;
468  } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
469  lowerCover(IPC);
470  MadeChange = true;
471  } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
472  lowerValueProfileInst(IPVP);
473  MadeChange = true;
474  }
475  }
476  }
477 
478  if (!MadeChange)
479  return false;
480 
481  promoteCounterLoadStores(F);
482  return true;
483 }
484 
485 bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
486  // Mach-O don't support weak external references.
487  if (TT.isOSBinFormatMachO())
488  return false;
489 
490  if (RuntimeCounterRelocation.getNumOccurrences() > 0)
491  return RuntimeCounterRelocation;
492 
493  // Fuchsia uses runtime counter relocation by default.
494  return TT.isOSFuchsia();
495 }
496 
497 bool InstrProfiling::isCounterPromotionEnabled() const {
498  if (DoCounterPromotion.getNumOccurrences() > 0)
499  return DoCounterPromotion;
500 
501  return Options.DoCounterPromotion;
502 }
503 
504 void InstrProfiling::promoteCounterLoadStores(Function *F) {
505  if (!isCounterPromotionEnabled())
506  return;
507 
508  DominatorTree DT(*F);
509  LoopInfo LI(DT);
510  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
511 
512  std::unique_ptr<BlockFrequencyInfo> BFI;
513  if (Options.UseBFIInPromotion) {
514  std::unique_ptr<BranchProbabilityInfo> BPI;
515  BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
516  BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
517  }
518 
519  for (const auto &LoadStore : PromotionCandidates) {
520  auto *CounterLoad = LoadStore.first;
521  auto *CounterStore = LoadStore.second;
522  BasicBlock *BB = CounterLoad->getParent();
523  Loop *ParentLoop = LI.getLoopFor(BB);
524  if (!ParentLoop)
525  continue;
526  LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
527  }
528 
530 
531  // Do a post-order traversal of the loops so that counter updates can be
532  // iteratively hoisted outside the loop nest.
533  for (auto *Loop : llvm::reverse(Loops)) {
534  PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
535  Promoter.run(&TotalCountersPromoted);
536  }
537 }
538 
539 static bool needsRuntimeHookUnconditionally(const Triple &TT) {
540  // On Fuchsia, we only need runtime hook if any counters are present.
541  if (TT.isOSFuchsia())
542  return false;
543 
544  return true;
545 }
546 
547 /// Check if the module contains uses of any profiling intrinsics.
549  auto containsIntrinsic = [&](int ID) {
550  if (auto *F = M.getFunction(Intrinsic::getName(ID)))
551  return !F->use_empty();
552  return false;
553  };
554  return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
555  containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
556  containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
557  containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
558 }
559 
561  Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
562  this->M = &M;
563  this->GetTLI = std::move(GetTLI);
564  NamesVar = nullptr;
565  NamesSize = 0;
566  ProfileDataMap.clear();
567  CompilerUsedVars.clear();
568  UsedVars.clear();
569  TT = Triple(M.getTargetTriple());
570 
571  bool MadeChange = false;
572 
573  // Emit the runtime hook even if no counters are present.
575  MadeChange = emitRuntimeHook();
576 
577  // Improve compile time by avoiding linear scans when there is no work.
578  GlobalVariable *CoverageNamesVar =
579  M.getNamedGlobal(getCoverageUnusedNamesVarName());
580  if (!containsProfilingIntrinsics(M) && !CoverageNamesVar)
581  return MadeChange;
582 
583  // We did not know how many value sites there would be inside
584  // the instrumented function. This is counting the number of instrumented
585  // target value sites to enter it as field in the profile data variable.
586  for (Function &F : M) {
587  InstrProfIncrementInst *FirstProfIncInst = nullptr;
588  for (BasicBlock &BB : F)
589  for (auto I = BB.begin(), E = BB.end(); I != E; I++)
590  if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
591  computeNumValueSiteCounts(Ind);
592  else if (FirstProfIncInst == nullptr)
593  FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
594 
595  // Value profiling intrinsic lowering requires per-function profile data
596  // variable to be created first.
597  if (FirstProfIncInst != nullptr)
598  static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
599  }
600 
601  for (Function &F : M)
602  MadeChange |= lowerIntrinsics(&F);
603 
604  if (CoverageNamesVar) {
605  lowerCoverageData(CoverageNamesVar);
606  MadeChange = true;
607  }
608 
609  if (!MadeChange)
610  return false;
611 
612  emitVNodes();
613  emitNameData();
614  emitRuntimeHook();
615  emitRegistration();
616  emitUses();
617  emitInitialization();
618  return true;
619 }
620 
622  Module &M, const TargetLibraryInfo &TLI,
623  ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
624  LLVMContext &Ctx = M.getContext();
625  auto *ReturnTy = Type::getVoidTy(M.getContext());
626 
628  if (auto AK = TLI.getExtAttrForI32Param(false))
629  AL = AL.addParamAttribute(M.getContext(), 2, AK);
630 
632  CallType == ValueProfilingCallType::MemOp) &&
633  "Must be Default or MemOp");
634  Type *ParamTypes[] = {
635 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
637  };
638  auto *ValueProfilingCallTy =
639  FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
640  StringRef FuncName = CallType == ValueProfilingCallType::Default
643  return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
644 }
645 
646 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
647  GlobalVariable *Name = Ind->getName();
649  uint64_t Index = Ind->getIndex()->getZExtValue();
650  auto &PD = ProfileDataMap[Name];
651  PD.NumValueSites[ValueKind] =
652  std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));
653 }
654 
655 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
656  // TODO: Value profiling heavily depends on the data section which is omitted
657  // in lightweight mode. We need to move the value profile pointer to the
658  // Counter struct to get this working.
659  assert(
661  "Value profiling is not yet supported with lightweight instrumentation");
662  GlobalVariable *Name = Ind->getName();
663  auto It = ProfileDataMap.find(Name);
664  assert(It != ProfileDataMap.end() && It->second.DataVar &&
665  "value profiling detected in function with no counter incerement");
666 
667  GlobalVariable *DataVar = It->second.DataVar;
669  uint64_t Index = Ind->getIndex()->getZExtValue();
670  for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
671  Index += It->second.NumValueSites[Kind];
672 
673  IRBuilder<> Builder(Ind);
674  bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
675  llvm::InstrProfValueKind::IPVK_MemOPSize);
676  CallInst *Call = nullptr;
677  auto *TLI = &GetTLI(*Ind->getFunction());
678 
679  // To support value profiling calls within Windows exception handlers, funclet
680  // information contained within operand bundles needs to be copied over to
681  // the library call. This is required for the IR to be processed by the
682  // WinEHPrepare pass.
684  Ind->getOperandBundlesAsDefs(OpBundles);
685  if (!IsMemOpSize) {
686  Value *Args[3] = {Ind->getTargetValue(),
687  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
688  Builder.getInt32(Index)};
689  Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
690  OpBundles);
691  } else {
692  Value *Args[3] = {Ind->getTargetValue(),
693  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
694  Builder.getInt32(Index)};
695  Call = Builder.CreateCall(
696  getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
697  Args, OpBundles);
698  }
699  if (auto AK = TLI->getExtAttrForI32Param(false))
700  Call->addParamAttr(2, AK);
701  Ind->replaceAllUsesWith(Call);
702  Ind->eraseFromParent();
703 }
704 
705 Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) {
706  auto *Counters = getOrCreateRegionCounters(I);
708 
709  auto *Addr = Builder.CreateConstInBoundsGEP2_32(
710  Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());
711 
712  if (!isRuntimeCounterRelocationEnabled())
713  return Addr;
714 
715  Type *Int64Ty = Type::getInt64Ty(M->getContext());
716  Function *Fn = I->getParent()->getParent();
717  LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
718  if (!BiasLI) {
719  IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
720  auto *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
721  if (!Bias) {
722  // Compiler must define this variable when runtime counter relocation
723  // is being used. Runtime has a weak external reference that is used
724  // to check whether that's the case or not.
725  Bias = new GlobalVariable(
726  *M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
728  Bias->setVisibility(GlobalVariable::HiddenVisibility);
729  // A definition that's weak (linkonce_odr) without being in a COMDAT
730  // section wouldn't lead to link errors, but it would lead to a dead
731  // data word from every TU but one. Putting it in COMDAT ensures there
732  // will be exactly one data slot in the link.
733  if (TT.supportsCOMDAT())
734  Bias->setComdat(M->getOrInsertComdat(Bias->getName()));
735  }
736  BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
737  }
738  auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
739  return Builder.CreateIntToPtr(Add, Addr->getType());
740 }
741 
742 void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) {
743  auto *Addr = getCounterAddress(CoverInstruction);
744  IRBuilder<> Builder(CoverInstruction);
745  // We store zero to represent that this block is covered.
746  Builder.CreateStore(Builder.getInt8(0), Addr);
747  CoverInstruction->eraseFromParent();
748 }
749 
750 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
751  auto *Addr = getCounterAddress(Inc);
752 
753  IRBuilder<> Builder(Inc);
754  if (Options.Atomic || AtomicCounterUpdateAll ||
755  (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {
756  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
758  } else {
759  Value *IncStep = Inc->getStep();
760  Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
761  auto *Count = Builder.CreateAdd(Load, Inc->getStep());
762  auto *Store = Builder.CreateStore(Count, Addr);
763  if (isCounterPromotionEnabled())
764  PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
765  }
766  Inc->eraseFromParent();
767 }
768 
769 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
770  ConstantArray *Names =
771  cast<ConstantArray>(CoverageNamesVar->getInitializer());
772  for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
773  Constant *NC = Names->getOperand(I);
774  Value *V = NC->stripPointerCasts();
775  assert(isa<GlobalVariable>(V) && "Missing reference to function name");
776  GlobalVariable *Name = cast<GlobalVariable>(V);
777 
778  Name->setLinkage(GlobalValue::PrivateLinkage);
779  ReferencedNames.push_back(Name);
780  if (isa<ConstantExpr>(NC))
781  NC->dropAllReferences();
782  }
783  CoverageNamesVar->eraseFromParent();
784 }
785 
786 /// Get the name of a profiling variable for a particular function.
787 static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
788  bool &Renamed) {
789  StringRef NamePrefix = getInstrProfNameVarPrefix();
790  StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
791  Function *F = Inc->getParent()->getParent();
792  Module *M = F->getParent();
793  if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
794  !canRenameComdatFunc(*F)) {
795  Renamed = false;
796  return (Prefix + Name).str();
797  }
798  Renamed = true;
799  uint64_t FuncHash = Inc->getHash()->getZExtValue();
800  SmallVector<char, 24> HashPostfix;
801  if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
802  return (Prefix + Name).str();
803  return (Prefix + Name + "." + Twine(FuncHash)).str();
804 }
805 
807  auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
808  if (!MD)
809  return 0;
810 
811  // If the flag is a ConstantAsMetadata, it should be an integer representable
812  // in 64-bits.
813  return cast<ConstantInt>(MD->getValue())->getZExtValue();
814 }
815 
816 static bool enablesValueProfiling(const Module &M) {
817  return isIRPGOFlagSet(&M) ||
818  getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;
819 }
820 
821 // Conservatively returns true if data variables may be referenced by code.
822 static bool profDataReferencedByCode(const Module &M) {
823  return enablesValueProfiling(M);
824 }
825 
826 static inline bool shouldRecordFunctionAddr(Function *F) {
827  // Only record function addresses if IR PGO is enabled or if clang value
828  // profiling is enabled. Recording function addresses greatly increases object
829  // file size, because it prevents the inliner from deleting functions that
830  // have been inlined everywhere.
831  if (!profDataReferencedByCode(*F->getParent()))
832  return false;
833 
834  // Check the linkage
835  bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
836  if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
837  !HasAvailableExternallyLinkage)
838  return true;
839 
840  // A function marked 'alwaysinline' with available_externally linkage can't
841  // have its address taken. Doing so would create an undefined external ref to
842  // the function, which would fail to link.
843  if (HasAvailableExternallyLinkage &&
844  F->hasFnAttribute(Attribute::AlwaysInline))
845  return false;
846 
847  // Prohibit function address recording if the function is both internal and
848  // COMDAT. This avoids the profile data variable referencing internal symbols
849  // in COMDAT.
850  if (F->hasLocalLinkage() && F->hasComdat())
851  return false;
852 
853  // Check uses of this function for other than direct calls or invokes to it.
854  // Inline virtual functions have linkeOnceODR linkage. When a key method
855  // exists, the vtable will only be emitted in the TU where the key method
856  // is defined. In a TU where vtable is not available, the function won't
857  // be 'addresstaken'. If its address is not recorded here, the profile data
858  // with missing address may be picked by the linker leading to missing
859  // indirect call target info.
860  return F->hasAddressTaken() || F->hasLinkOnceLinkage();
861 }
862 
864  // Don't do this for Darwin. compiler-rt uses linker magic.
865  if (TT.isOSDarwin())
866  return false;
867  // Use linker script magic to get data/cnts/name start/end.
868  if (TT.isOSAIX() || TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
869  TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS() || TT.isOSWindows())
870  return false;
871 
872  return true;
873 }
874 
876 InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name,
877  GlobalValue::LinkageTypes Linkage) {
878  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
879  auto &Ctx = M->getContext();
880  GlobalVariable *GV;
881  if (isa<InstrProfCoverInst>(Inc)) {
882  auto *CounterTy = Type::getInt8Ty(Ctx);
883  auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);
884  // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
885  std::vector<Constant *> InitialValues(NumCounters,
886  Constant::getAllOnesValue(CounterTy));
887  GV = new GlobalVariable(*M, CounterArrTy, false, Linkage,
888  ConstantArray::get(CounterArrTy, InitialValues),
889  Name);
890  GV->setAlignment(Align(1));
891  } else {
892  auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
893  GV = new GlobalVariable(*M, CounterTy, false, Linkage,
894  Constant::getNullValue(CounterTy), Name);
895  GV->setAlignment(Align(8));
896  }
897  return GV;
898 }
899 
901 InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
902  GlobalVariable *NamePtr = Inc->getName();
903  auto &PD = ProfileDataMap[NamePtr];
904  if (PD.RegionCounters)
905  return PD.RegionCounters;
906 
907  // Match the linkage and visibility of the name global.
908  Function *Fn = Inc->getParent()->getParent();
910  GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
911 
912  // Use internal rather than private linkage so the counter variable shows up
913  // in the symbol table when using debug info for correlation.
915  Linkage == GlobalValue::PrivateLinkage)
917 
918  // Due to the limitation of binder as of 2021/09/28, the duplicate weak
919  // symbols in the same csect won't be discarded. When there are duplicate weak
920  // symbols, we can NOT guarantee that the relocations get resolved to the
921  // intended weak symbol, so we can not ensure the correctness of the relative
922  // CounterPtr, so we have to use private linkage for counter and data symbols.
923  if (TT.isOSBinFormatXCOFF()) {
925  Visibility = GlobalValue::DefaultVisibility;
926  }
927  // Move the name variable to the right section. Place them in a COMDAT group
928  // if the associated function is a COMDAT. This will make sure that only one
929  // copy of counters of the COMDAT function will be emitted after linking. Keep
930  // in mind that this pass may run before the inliner, so we need to create a
931  // new comdat group for the counters and profiling data. If we use the comdat
932  // of the parent function, that will result in relocations against discarded
933  // sections.
934  //
935  // If the data variable is referenced by code, counters and data have to be
936  // in different comdats for COFF because the Visual C++ linker will report
937  // duplicate symbol errors if there are multiple external symbols with the
938  // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
939  //
940  // For ELF, when not using COMDAT, put counters, data and values into a
941  // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
942  // allows -z start-stop-gc to discard the entire group when the function is
943  // discarded.
944  bool DataReferencedByCode = profDataReferencedByCode(*M);
945  bool NeedComdat = needsComdatForCounter(*Fn, *M);
946  bool Renamed;
947  std::string CntsVarName =
948  getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed);
949  std::string DataVarName =
950  getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
951  auto MaybeSetComdat = [&](GlobalVariable *GV) {
952  bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
953  if (UseComdat) {
954  StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
955  ? GV->getName()
956  : CntsVarName;
957  Comdat *C = M->getOrInsertComdat(GroupName);
958  if (!NeedComdat)
959  C->setSelectionKind(Comdat::NoDeduplicate);
960  GV->setComdat(C);
961  }
962  };
963 
964  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
965  LLVMContext &Ctx = M->getContext();
966 
967  auto *CounterPtr = createRegionCounters(Inc, CntsVarName, Linkage);
968  CounterPtr->setVisibility(Visibility);
969  CounterPtr->setSection(
970  getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
971  MaybeSetComdat(CounterPtr);
972  CounterPtr->setLinkage(Linkage);
973  PD.RegionCounters = CounterPtr;
974  if (DebugInfoCorrelate) {
975  if (auto *SP = Fn->getSubprogram()) {
976  DIBuilder DB(*M, true, SP->getUnit());
977  Metadata *FunctionNameAnnotation[] = {
980  };
981  Metadata *CFGHashAnnotation[] = {
984  };
985  Metadata *NumCountersAnnotation[] = {
988  };
989  auto Annotations = DB.getOrCreateArray({
990  MDNode::get(Ctx, FunctionNameAnnotation),
991  MDNode::get(Ctx, CFGHashAnnotation),
992  MDNode::get(Ctx, NumCountersAnnotation),
993  });
994  auto *DICounter = DB.createGlobalVariableExpression(
995  SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
996  /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),
997  CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
998  /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
999  Annotations);
1000  CounterPtr->addDebugInfo(DICounter);
1001  DB.finalize();
1002  } else {
1003  std::string Msg = ("Missing debug info for function " + Fn->getName() +
1004  "; required for profile correlation.")
1005  .str();
1006  Ctx.diagnose(
1007  DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
1008  }
1009  }
1010 
1011  auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
1012  // Allocate statically the array of pointers to value profile nodes for
1013  // the current function.
1014  Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
1015  uint64_t NS = 0;
1016  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1017  NS += PD.NumValueSites[Kind];
1018  if (NS > 0 && ValueProfileStaticAlloc &&
1020  ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
1021  auto *ValuesVar = new GlobalVariable(
1022  *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
1023  getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
1024  ValuesVar->setVisibility(Visibility);
1025  ValuesVar->setSection(
1026  getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
1027  ValuesVar->setAlignment(Align(8));
1028  MaybeSetComdat(ValuesVar);
1029  ValuesPtrExpr =
1031  }
1032 
1033  if (DebugInfoCorrelate) {
1034  // Mark the counter variable as used so that it isn't optimized out.
1035  CompilerUsedVars.push_back(PD.RegionCounters);
1036  return PD.RegionCounters;
1037  }
1038 
1039  // Create data variable.
1040  auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext());
1041  auto *Int16Ty = Type::getInt16Ty(Ctx);
1042  auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
1043  Type *DataTypes[] = {
1044 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1046  };
1047  auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
1048 
1049  Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
1050  ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
1051  : ConstantPointerNull::get(Int8PtrTy);
1052 
1053  Constant *Int16ArrayVals[IPVK_Last + 1];
1054  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1055  Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
1056 
1057  // If the data variable is not referenced by code (if we don't emit
1058  // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
1059  // data variable live under linker GC, the data variable can be private. This
1060  // optimization applies to ELF.
1061  //
1062  // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
1063  // to be false.
1064  //
1065  // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
1066  // that other copies must have the same CFG and cannot have value profiling.
1067  // If no hash suffix, other profd copies may be referenced by code.
1068  if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1069  (TT.isOSBinFormatELF() ||
1070  (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
1072  Visibility = GlobalValue::DefaultVisibility;
1073  }
1074  auto *Data =
1075  new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName);
1076  // Reference the counter variable with a label difference (link-time
1077  // constant).
1078  auto *RelativeCounterPtr =
1079  ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
1080  ConstantExpr::getPtrToInt(Data, IntPtrTy));
1081 
1082  Constant *DataVals[] = {
1083 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1085  };
1086  Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
1087 
1088  Data->setVisibility(Visibility);
1089  Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
1090  Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
1091  MaybeSetComdat(Data);
1092  Data->setLinkage(Linkage);
1093 
1094  PD.DataVar = Data;
1095 
1096  // Mark the data variable as used so that it isn't stripped out.
1097  CompilerUsedVars.push_back(Data);
1098  // Now that the linkage set by the FE has been passed to the data and counter
1099  // variables, reset Name variable's linkage and visibility to private so that
1100  // it can be removed later by the compiler.
1102  // Collect the referenced names to be used by emitNameData.
1103  ReferencedNames.push_back(NamePtr);
1104 
1105  return PD.RegionCounters;
1106 }
1107 
1108 void InstrProfiling::emitVNodes() {
1109  if (!ValueProfileStaticAlloc)
1110  return;
1111 
1112  // For now only support this on platforms that do
1113  // not require runtime registration to discover
1114  // named section start/end.
1116  return;
1117 
1118  size_t TotalNS = 0;
1119  for (auto &PD : ProfileDataMap) {
1120  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1121  TotalNS += PD.second.NumValueSites[Kind];
1122  }
1123 
1124  if (!TotalNS)
1125  return;
1126 
1127  uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1128 // Heuristic for small programs with very few total value sites.
1129 // The default value of vp-counters-per-site is chosen based on
1130 // the observation that large apps usually have a low percentage
1131 // of value sites that actually have any profile data, and thus
1132 // the average number of counters per site is low. For small
1133 // apps with very few sites, this may not be true. Bump up the
1134 // number of counters in this case.
1135 #define INSTR_PROF_MIN_VAL_COUNTS 10
1136  if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
1137  NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
1138 
1139  auto &Ctx = M->getContext();
1140  Type *VNodeTypes[] = {
1141 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1143  };
1144  auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
1145 
1146  ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
1147  auto *VNodesVar = new GlobalVariable(
1148  *M, VNodesTy, false, GlobalValue::PrivateLinkage,
1150  VNodesVar->setSection(
1151  getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
1152  // VNodesVar is used by runtime but not referenced via relocation by other
1153  // sections. Conservatively make it linker retained.
1154  UsedVars.push_back(VNodesVar);
1155 }
1156 
1157 void InstrProfiling::emitNameData() {
1158  std::string UncompressedData;
1159 
1160  if (ReferencedNames.empty())
1161  return;
1162 
1163  std::string CompressedNameStr;
1164  if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
1167  }
1168 
1169  auto &Ctx = M->getContext();
1170  auto *NamesVal =
1171  ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false);
1172  NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
1173  GlobalValue::PrivateLinkage, NamesVal,
1175  NamesSize = CompressedNameStr.size();
1176  NamesVar->setSection(
1177  getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
1178  // On COFF, it's important to reduce the alignment down to 1 to prevent the
1179  // linker from inserting padding before the start of the names section or
1180  // between names entries.
1181  NamesVar->setAlignment(Align(1));
1182  // NamesVar is used by runtime but not referenced via relocation by other
1183  // sections. Conservatively make it linker retained.
1184  UsedVars.push_back(NamesVar);
1185 
1186  for (auto *NamePtr : ReferencedNames)
1187  NamePtr->eraseFromParent();
1188 }
1189 
1190 void InstrProfiling::emitRegistration() {
1192  return;
1193 
1194  // Construct the function.
1195  auto *VoidTy = Type::getVoidTy(M->getContext());
1196  auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
1197  auto *Int64Ty = Type::getInt64Ty(M->getContext());
1198  auto *RegisterFTy = FunctionType::get(VoidTy, false);
1199  auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1201  RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1202  if (Options.NoRedZone)
1203  RegisterF->addFnAttr(Attribute::NoRedZone);
1204 
1205  auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1206  auto *RuntimeRegisterF =
1209 
1210  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
1211  for (Value *Data : CompilerUsedVars)
1212  if (!isa<Function>(Data))
1213  IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1214  for (Value *Data : UsedVars)
1215  if (Data != NamesVar && !isa<Function>(Data))
1216  IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1217 
1218  if (NamesVar) {
1219  Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1220  auto *NamesRegisterTy =
1221  FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
1222  auto *NamesRegisterF =
1225  IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
1226  IRB.getInt64(NamesSize)});
1227  }
1228 
1229  IRB.CreateRetVoid();
1230 }
1231 
1232 bool InstrProfiling::emitRuntimeHook() {
1233  // We expect the linker to be invoked with -u<hook_var> flag for Linux
1234  // in which case there is no need to emit the external variable.
1235  if (TT.isOSLinux())
1236  return false;
1237 
1238  // If the module's provided its own runtime, we don't need to do anything.
1239  if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
1240  return false;
1241 
1242  // Declare an external variable that will pull in the runtime initialization.
1243  auto *Int32Ty = Type::getInt32Ty(M->getContext());
1244  auto *Var =
1246  nullptr, getInstrProfRuntimeHookVarName());
1247 
1248  if (TT.isOSBinFormatELF() && !TT.isPS()) {
1249  // Mark the user variable as used so that it isn't stripped out.
1250  CompilerUsedVars.push_back(Var);
1251  } else {
1252  // Make a function that uses it.
1256  User->addFnAttr(Attribute::NoInline);
1257  if (Options.NoRedZone)
1258  User->addFnAttr(Attribute::NoRedZone);
1259  User->setVisibility(GlobalValue::HiddenVisibility);
1260  if (TT.supportsCOMDAT())
1261  User->setComdat(M->getOrInsertComdat(User->getName()));
1262 
1263  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
1264  auto *Load = IRB.CreateLoad(Int32Ty, Var);
1265  IRB.CreateRet(Load);
1266 
1267  // Mark the function as used so that it isn't stripped out.
1268  CompilerUsedVars.push_back(User);
1269  }
1270  return true;
1271 }
1272 
1273 void InstrProfiling::emitUses() {
1274  // The metadata sections are parallel arrays. Optimizers (e.g.
1275  // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
1276  // we conservatively retain all unconditionally in the compiler.
1277  //
1278  // On ELF and Mach-O, the linker can guarantee the associated sections will be
1279  // retained or discarded as a unit, so llvm.compiler.used is sufficient.
1280  // Similarly on COFF, if prof data is not referenced by code we use one comdat
1281  // and ensure this GC property as well. Otherwise, we have to conservatively
1282  // make all of the sections retained by the linker.
1283  if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
1285  appendToCompilerUsed(*M, CompilerUsedVars);
1286  else
1287  appendToUsed(*M, CompilerUsedVars);
1288 
1289  // We do not add proper references from used metadata sections to NamesVar and
1290  // VNodesVar, so we have to be conservative and place them in llvm.used
1291  // regardless of the target,
1292  appendToUsed(*M, UsedVars);
1293 }
1294 
1295 void InstrProfiling::emitInitialization() {
1296  // Create ProfileFileName variable. Don't don't this for the
1297  // context-sensitive instrumentation lowering: This lowering is after
1298  // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1299  // have already create the variable before LTO/ThinLTO linking.
1300  if (!IsCS)
1301  createProfileFileNameVar(*M, Options.InstrProfileOutput);
1302  Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1303  if (!RegisterF)
1304  return;
1305 
1306  // Create the initialization function.
1307  auto *VoidTy = Type::getVoidTy(M->getContext());
1308  auto *F = Function::Create(FunctionType::get(VoidTy, false),
1311  F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1312  F->addFnAttr(Attribute::NoInline);
1313  if (Options.NoRedZone)
1314  F->addFnAttr(Attribute::NoRedZone);
1315 
1316  // Add the basic block and the necessary calls.
1317  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1318  IRB.CreateCall(RegisterF, {});
1319  IRB.CreateRetVoid();
1320 
1321  appendToGlobalCtors(*M, F, 0);
1322 }
llvm::createInstrProfilingLegacyPass
ModulePass * createInstrProfilingLegacyPass(const InstrProfOptions &Options=InstrProfOptions(), bool IsCS=false)
Insert frontend instrumentation based profiling. Parameter IsCS indicates if.
Definition: InstrProfiling.cpp:452
INSTR_PROF_MIN_VAL_COUNTS
#define INSTR_PROF_MIN_VAL_COUNTS
i
i
Definition: README.txt:29
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
Int32Ty
IntegerType * Int32Ty
Definition: NVVMIntrRange.cpp:67
llvm::GlobalVariable::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:451
instrumentation
xray instrumentation
Definition: XRayInstrumentation.cpp:268
llvm::GlobalObject::setComdat
void setComdat(Comdat *C)
Definition: Globals.cpp:194
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::CallBase::getOperandBundlesAsDefs
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition: Instructions.cpp:396
llvm::InstrProfIncrementInst::getStep
Value * getStep() const
Definition: IntrinsicInst.cpp:191
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(InstrProfilingLegacyPass, "instrprof", "Frontend instrumentation-based coverage lowering.", false, false) INITIALIZE_PASS_END(InstrProfilingLegacyPass
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::AArch64CC::AL
@ AL
Definition: AArch64BaseInfo.h:269
llvm::LoopBase::getExitBlocks
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
Definition: LoopInfoImpl.h:61
needsRuntimeHookUnconditionally
static bool needsRuntimeHookUnconditionally(const Triple &TT)
Definition: InstrProfiling.cpp:539
llvm::StructType::get
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:406
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:291
llvm::getInstrProfNamesVarName
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition: InstrProf.h:101
getVarName
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
Definition: InstrProfiling.cpp:787
llvm::ModulePass
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:248
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:104
IntrinsicInst.h
llvm::DIBuilder
Definition: DIBuilder.h:41
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:780
llvm::Triple::isOSBinFormatCOFF
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
Definition: Triple.h:661
llvm::cl::Prefix
@ Prefix
Definition: CommandLine.h:160
llvm::InstrProfCorrelator::NumCountersAttributeName
static const char * NumCountersAttributeName
Definition: InstrProfCorrelator.h:55
llvm::LoadAndStorePromoter
Helper class for promoting a collection of loads and stores into SSA Form using the SSAUpdater.
Definition: SSAUpdater.h:136
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition: GlobalValue.h:509
llvm::GlobalValue::HiddenVisibility
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
llvm::Function
Definition: Function.h:60
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:546
StringRef.h
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1306
llvm::NVPTX::LoadStore
LoadStore
Definition: NVPTX.h:99
Pass.h
llvm::InstrProfInstBase
A base class for all instrprof intrinsics.
Definition: IntrinsicInst.h:1235
Loops
Hexagon Hardware Loops
Definition: HexagonHardwareLoops.cpp:372
llvm::Triple::isPS
bool isPS() const
Tests whether the target is the PS4 or PS5 platform.
Definition: Triple.h:698
llvm::needsComdatForCounter
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1111
llvm::Comdat::NoDeduplicate
@ NoDeduplicate
No deduplication is performed.
Definition: Comdat.h:39
llvm::X86II::PD
@ PD
Definition: X86BaseInfo.h:787
llvm::getPGOFuncNameVarInitializer
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:477
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
llvm::Function::getSubprogram
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1639
ErrorHandling.h
llvm::Function::getEntryBlock
const BasicBlock & getEntryBlock() const
Definition: Function.h:710
llvm::Intrinsic::getName
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:880
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:39
llvm::ConstantExpr::getBitCast
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2199
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:361
Error.h
llvm::AtomicOrdering::SequentiallyConsistent
@ SequentiallyConsistent
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
FAM
FunctionAnalysisManager FAM
Definition: PassBuilderBindings.cpp:59
llvm::getInstrProfRuntimeHookVarName
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:149
llvm::MemOp
Definition: TargetLowering.h:111
InstrCount
static unsigned InstrCount
Definition: DFAPacketizer.cpp:53
llvm::PreservedAnalyses::none
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
containsProfilingIntrinsics
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
Definition: InstrProfiling.cpp:548
Module.h
llvm::reverse
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
Definition: STLExtras.h:380
llvm::AttributeList
Definition: Attributes.h:425
llvm::TargetLibraryInfo::getExtAttrForI32Param
Attribute::AttrKind getExtAttrForI32Param(bool Signed=true) const
Returns extension attribute kind to be used for i32 parameters corresponding to C-level int or unsign...
Definition: TargetLibraryInfo.h:387
llvm::getInstrProfNamesRegFuncName
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:136
llvm::InstrProfInstBase::getIndex
ConstantInt * getIndex() const
Definition: IntrinsicInst.cpp:185
llvm::InstrProfCorrelator::CFGHashAttributeName
static const char * CFGHashAttributeName
Definition: InstrProfCorrelator.h:54
llvm::Triple::isOSBinFormatXCOFF
bool isOSBinFormatXCOFF() const
Tests whether the OS uses the XCOFF binary format.
Definition: Triple.h:679
llvm::GlobalValue::UnnamedAddr::Global
@ Global
llvm::InstrProfInstBase::getNumCounters
ConstantInt * getNumCounters() const
Definition: IntrinsicInst.cpp:179
llvm::GlobalValue::DefaultVisibility
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:63
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:420
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
llvm::Triple::isOSLinux
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:619
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::InstrProfInstBase::getHash
ConstantInt * getHash() const
Definition: IntrinsicInst.h:1243
llvm::GlobalValue::LinkageTypes
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
llvm::ArrayType
Class to represent array types.
Definition: DerivedTypes.h:357
llvm::createProfileFileNameVar
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1182
llvm::Type::getInt8Ty
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:237
llvm::Data
@ Data
Definition: SIMachineScheduler.h:55
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:239
llvm::SPIRV::ImageOperand::Bias
@ Bias
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1384
llvm::InstrProfiling
Instrumentation based profiling lowering pass.
Definition: InstrProfiling.h:34
F
#define F(x, y, z)
Definition: MD5.cpp:55
InstrProfData.inc
llvm::AtomicOrdering::Monotonic
@ Monotonic
getIntModuleFlagOrZero
static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag)
Definition: InstrProfiling.cpp:806
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::Annotations
Annotations lets you mark points and ranges inside source code, for tests:
Definition: Annotations.h:48
llvm::InstrProfValueProfileInst::getValueKind
ConstantInt * getValueKind() const
Definition: IntrinsicInst.h:1300
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ConstantExpr::getSub
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2675
llvm::StringRef::substr
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:615
llvm::ConstantPointerNull
A constant pointer value that points to null.
Definition: Constants.h:535
Instruction.h
CommandLine.h
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::AMDGPU::HSAMD::ValueKind
ValueKind
Value kinds.
Definition: AMDGPUMetadata.h:78
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition: BlockFrequencyInfo.h:37
llvm::Triple::isOSBinFormatELF
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Definition: Triple.h:656
GlobalValue.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::GlobalValue::VisibilityTypes
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:62
Constants.h
InstrProfiling.h
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::getInstrProfValuesVarPrefix
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:94
llvm::DS_Warning
@ DS_Warning
Definition: DiagnosticInfo.h:51
llvm::GlobalObject::setSection
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:248
llvm::User
Definition: User.h:44
instrprof
instrprof
Definition: InstrProfiling.cpp:447
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::InstrProfCorrelator::FunctionNameAttributeName
static const char * FunctionNameAttributeName
Definition: InstrProfCorrelator.h:53
Twine.h
llvm::InstrProfValueProfileInst::getTargetValue
Value * getTargetValue() const
Definition: IntrinsicInst.h:1296
llvm::BranchProbabilityInfo
Analysis providing branch probability information.
Definition: BranchProbabilityInfo.h:113
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::ConstantExpr::getPtrToInt
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2171
llvm::canRenameComdatFunc
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1159
TargetLibraryInfo.h
llvm::getInstrProfCountersVarPrefix
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:91
llvm::Triple::supportsCOMDAT
bool supportsCOMDAT() const
Tests whether the target supports comdat.
Definition: Triple.h:916
llvm::BasicBlock::getFirstInsertionPt
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:246
false
Definition: StackSlotColoring.cpp:141
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
llvm::dwarf::toStringRef
StringRef toStringRef(const Optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
Definition: DWARFFormValue.h:193
llvm::getInstrProfInitFuncName
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:144
llvm::ConstantArray
ConstantArray - Constant Array Declarations.
Definition: Constants.h:410
llvm::Constant::getAllOnesValue
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:395
llvm::Triple::isOSBinFormatMachO
bool isOSBinFormatMachO() const
Tests whether the environment is MachO.
Definition: Triple.h:669
llvm::Instruction
Definition: Instruction.h:42
llvm::MCID::Flag
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:147
InstrProf.h
llvm::Triple::getObjectFormat
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
Definition: Triple.h:371
llvm::appendToCompilerUsed
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
Definition: ModuleUtils.cpp:109
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:143
Options
const char LLVMTargetMachineRef LLVMPassBuilderOptionsRef Options
Definition: PassBuilderBindings.cpp:48
llvm::LoopBase::getExitingBlocks
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
Definition: LoopInfoImpl.h:33
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
llvm::InstrProfValueProfileInst::getIndex
ConstantInt * getIndex() const
Definition: IntrinsicInst.h:1305
Align
uint64_t Align
Definition: ELFObjHandler.cpp:81
llvm::GlobalValue::InternalLinkage
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
llvm::Comdat
Definition: Comdat.h:33
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:155
llvm::Metadata
Root of the metadata hierarchy.
Definition: Metadata.h:62
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:730
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
Type.h
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
llvm::dxil::PointerTypeAnalysis::run
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
Definition: PointerTypeAnalysis.cpp:101
LoopInfo.h
llvm::Triple::isOSFuchsia
bool isOSFuchsia() const
Definition: Triple.h:538
llvm::getInstrProfVNodesVarName
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:97
llvm::getInstrProfNameVarPrefix
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:85
BasicBlock.h
llvm::cl::opt< bool >
SSA
Memory SSA
Definition: MemorySSA.cpp:73
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition: GlobalVariable.h:135
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:77
BranchProbabilityInfo.h
llvm::InstrProfValueProfileInst
This represents the llvm.instrprof.value.profile intrinsic.
Definition: IntrinsicInst.h:1287
llvm::TargetLibraryInfoWrapperPass
Definition: TargetLibraryInfo.h:468
uint64_t
llvm::GlobalValue::getVisibility
VisibilityTypes getVisibility() const
Definition: GlobalValue.h:234
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:78
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
llvm::ConstantPointerNull::get
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1696
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
profDataReferencedByCode
static bool profDataReferencedByCode(const Module &M)
Definition: InstrProfiling.cpp:822
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::DenseMap
Definition: DenseMap.h:716
shouldRecordFunctionAddr
static bool shouldRecordFunctionAddr(Function *F)
Definition: InstrProfiling.cpp:826
llvm::isIRPGOFlagSet
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1137
llvm::LoopInfoBase::getLoopFor
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:986
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
llvm::appendToUsed
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
Definition: ModuleUtils.cpp:105
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:618
DIBuilder.h
llvm::LoopBase::getLoopPreheader
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:166
llvm::GlobalValue::setLinkage
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:502
ArrayRef.h
llvm::InstrProfOptions
Options for the frontend instrumentation based profiling pass.
Definition: Instrumentation.h:110
llvm::MDString::get
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:497
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:137
llvm::DiagnosticInfoPGOProfile
Diagnostic information for the PGO profiler.
Definition: DiagnosticInfo.h:330
llvm::LoopBase::hasDedicatedExits
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
Definition: LoopInfoImpl.h:91
llvm::getInstrProfRuntimeHookVarUseFuncName
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition: InstrProf.h:155
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::InstrProfInstBase::getName
GlobalVariable * getName() const
Definition: IntrinsicInst.h:1238
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:82
lowering
Frontend instrumentation based coverage lowering
Definition: InstrProfiling.cpp:448
llvm::ArrayType::get
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:638
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
llvm::collectPGOFuncNameStrings
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:440
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
Triple.h
llvm::getInstrProfRegFuncsName
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:125
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::LoopInfo
Definition: LoopInfo.h:1102
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1624
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:263
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
if
if(llvm_vc STREQUAL "") set(fake_version_inc "$
Definition: CMakeLists.txt:14
llvm::Instruction::getFunction
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:69
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:529
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:97
uint32_t
llvm::ConstantDataArray::getString
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:3021
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::DebugInfoCorrelate
cl::opt< bool > DebugInfoCorrelate("debug-info-correlate", cl::desc("Use debug info to correlate profiles."), cl::init(false))
Definition: PGOInstrumentation.cpp:299
SSAUpdater.h
BlockFrequencyInfo.h
llvm::AMDGPUISD::BFI
@ BFI
Definition: AMDGPUISelLowering.h:429
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:305
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:173
llvm::Init
Definition: Record.h:281
llvm::AMDGPU::SendMsg::Msg
const CustomOperand< const MCSubtargetInfo & > Msg[]
Definition: AMDGPUAsmUtils.cpp:39
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:142
llvm::BasicBlock::front
const Instruction & front() const
Definition: BasicBlock.h:309
llvm::DoInstrProfNameCompression
cl::opt< bool > DoInstrProfNameCompression
llvm::LoopInfoBase::getLoopsInPreorder
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
Definition: LoopInfoImpl.h:575
Attributes.h
llvm::StringRef::size
constexpr LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:157
Constant.h
llvm::LLVMContext::diagnose
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Definition: LLVMContext.cpp:243
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:240
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::Constant::getNullValue
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:350
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
NC
#define NC
Definition: regutils.h:42
llvm::toString
const char * toString(DWARFSectionKind Kind)
Definition: DWARFUnitIndex.h:67
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
llvm::Constant::isZeroValue
bool isZeroValue() const
Return true if the value is negative zero or null value.
Definition: Constants.cpp:62
llvm::Error
Lightweight error class with error context and mandatory checking.
Definition: Error.h:155
GlobalVariable.h
getOrInsertValueProfilingCall
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
Definition: InstrProfiling.cpp:621
Casting.h
DiagnosticInfo.h
Function.h
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:774
enablesValueProfiling
static bool enablesValueProfiling(const Module &M)
Definition: InstrProfiling.cpp:816
llvm::TargetLibraryInfo
Provides information about what library functions are available for the current target.
Definition: TargetLibraryInfo.h:222
llvm::TargetStackID::Default
@ Default
Definition: TargetFrameLowering.h:28
llvm::ConstantArray::get
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1241
llvm::InstrProfiling::run
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Definition: InstrProfiling.cpp:430
llvm::InstrProfCoverInst
This represents the llvm.instrprof.cover intrinsic.
Definition: IntrinsicInst.h:1253
llvm::MCID::Add
@ Add
Definition: MCInstrDesc.h:185
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:475
llvm::InstrProfIncrementInst
This represents the llvm.instrprof.increment intrinsic.
Definition: IntrinsicInst.h:1264
llvm::getInstrProfSectionName
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:214
llvm::GlobalValue::ExternalLinkage
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:222
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Instructions.h
llvm::getInstrProfValueProfMemOpFuncName
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition: InstrProf.h:80
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
needsRuntimeRegistrationOfSectionRange
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
Definition: InstrProfiling.cpp:863
SmallVector.h
Dominators.h
llvm::getCoverageUnusedNamesVarName
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:116
ModuleUtils.h
llvm::getInstrProfCounterBiasVarName
StringRef getInstrProfCounterBiasVarName()
Definition: InstrProf.h:159
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::FunctionCallee
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:119
llvm::SmallVectorImpl< BasicBlock * >
InstrProfCorrelator.h
llvm::Type::getInt16Ty
static IntegerType * getInt16Ty(LLVMContext &C)
Definition: Type.cpp:238
DerivedTypes.h
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
llvm::InnerAnalysisManagerProxy
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:937
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1461
llvm::getInstrProfValueProfFuncName
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:74
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:394
llvm::appendToGlobalCtors
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:65
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::getInstrProfDataVarPrefix
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:88
llvm::cl::desc
Definition: CommandLine.h:405
llvm::GlobalObject::setAlignment
void setAlignment(MaybeAlign Align)
Definition: Globals.cpp:126
llvm::SSAUpdater
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition: SSAUpdater.h:38
llvm::initializeInstrProfilingLegacyPassPass
void initializeInstrProfilingLegacyPassPass(PassRegistry &)
llvm::GlobalValue::LinkOnceODRLinkage
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
InitializePasses.h
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::getInstrProfRegFuncName
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:131
llvm::TargetLibraryAnalysis
Analysis pass providing the TargetLibraryInfo.
Definition: TargetLibraryInfo.h:443
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38