LLVM  16.0.0git
InstrProfiling.cpp
Go to the documentation of this file.
1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10 // It also builds the data structures and initialization code needed for
11 // updating execution counts and emitting the profile at runtime.
12 //
13 //===----------------------------------------------------------------------===//
14 
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/ADT/Twine.h"
23 #include "llvm/Analysis/LoopInfo.h"
25 #include "llvm/IR/Attributes.h"
26 #include "llvm/IR/BasicBlock.h"
27 #include "llvm/IR/Constant.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DIBuilder.h"
30 #include "llvm/IR/DerivedTypes.h"
31 #include "llvm/IR/DiagnosticInfo.h"
32 #include "llvm/IR/Dominators.h"
33 #include "llvm/IR/Function.h"
34 #include "llvm/IR/GlobalValue.h"
35 #include "llvm/IR/GlobalVariable.h"
36 #include "llvm/IR/IRBuilder.h"
37 #include "llvm/IR/Instruction.h"
38 #include "llvm/IR/Instructions.h"
39 #include "llvm/IR/IntrinsicInst.h"
40 #include "llvm/IR/Module.h"
41 #include "llvm/IR/Type.h"
42 #include "llvm/InitializePasses.h"
43 #include "llvm/Pass.h"
46 #include "llvm/Support/Casting.h"
48 #include "llvm/Support/Error.h"
52 #include <algorithm>
53 #include <cassert>
54 #include <cstdint>
55 #include <string>
56 
57 using namespace llvm;
58 
59 #define DEBUG_TYPE "instrprof"
60 
61 namespace llvm {
63  DebugInfoCorrelate("debug-info-correlate",
64  cl::desc("Use debug info to correlate profiles."),
65  cl::init(false));
66 } // namespace llvm
67 
68 namespace {
69 
70 cl::opt<bool> DoHashBasedCounterSplit(
71  "hash-based-counter-split",
72  cl::desc("Rename counter variable of a comdat function based on cfg hash"),
73  cl::init(true));
74 
76  RuntimeCounterRelocation("runtime-counter-relocation",
77  cl::desc("Enable relocating counters at runtime."),
78  cl::init(false));
79 
80 cl::opt<bool> ValueProfileStaticAlloc(
81  "vp-static-alloc",
82  cl::desc("Do static counter allocation for value profiler"),
83  cl::init(true));
84 
85 cl::opt<double> NumCountersPerValueSite(
86  "vp-counters-per-site",
87  cl::desc("The average number of profile counters allocated "
88  "per value profiling site."),
89  // This is set to a very small value because in real programs, only
90  // a very small percentage of value sites have non-zero targets, e.g, 1/30.
91  // For those sites with non-zero profile, the average number of targets
92  // is usually smaller than 2.
93  cl::init(1.0));
94 
95 cl::opt<bool> AtomicCounterUpdateAll(
96  "instrprof-atomic-counter-update-all",
97  cl::desc("Make all profile counter updates atomic (for testing only)"),
98  cl::init(false));
99 
100 cl::opt<bool> AtomicCounterUpdatePromoted(
101  "atomic-counter-update-promoted",
102  cl::desc("Do counter update using atomic fetch add "
103  " for promoted counters only"),
104  cl::init(false));
105 
106 cl::opt<bool> AtomicFirstCounter(
107  "atomic-first-counter",
108  cl::desc("Use atomic fetch add for first counter in a function (usually "
109  "the entry counter)"),
110  cl::init(false));
111 
112 // If the option is not specified, the default behavior about whether
113 // counter promotion is done depends on how instrumentaiton lowering
114 // pipeline is setup, i.e., the default value of true of this option
115 // does not mean the promotion will be done by default. Explicitly
116 // setting this option can override the default behavior.
117 cl::opt<bool> DoCounterPromotion("do-counter-promotion",
118  cl::desc("Do counter register promotion"),
119  cl::init(false));
120 cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
121  "max-counter-promotions-per-loop", cl::init(20),
122  cl::desc("Max number counter promotions per loop to avoid"
123  " increasing register pressure too much"));
124 
125 // A debug option
127  MaxNumOfPromotions("max-counter-promotions", cl::init(-1),
128  cl::desc("Max number of allowed counter promotions"));
129 
130 cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
131  "speculative-counter-promotion-max-exiting", cl::init(3),
132  cl::desc("The max number of exiting blocks of a loop to allow "
133  " speculative counter promotion"));
134 
135 cl::opt<bool> SpeculativeCounterPromotionToLoop(
136  "speculative-counter-promotion-to-loop",
137  cl::desc("When the option is false, if the target block is in a loop, "
138  "the promotion will be disallowed unless the promoted counter "
139  " update can be further/iteratively promoted into an acyclic "
140  " region."));
141 
142 cl::opt<bool> IterativeCounterPromotion(
143  "iterative-counter-promotion", cl::init(true),
144  cl::desc("Allow counter promotion across the whole loop nest."));
145 
146 cl::opt<bool> SkipRetExitBlock(
147  "skip-ret-exit-block", cl::init(true),
148  cl::desc("Suppress counter promotion if exit blocks contain ret."));
149 
150 ///
151 /// A helper class to promote one counter RMW operation in the loop
152 /// into register update.
153 ///
154 /// RWM update for the counter will be sinked out of the loop after
155 /// the transformation.
156 ///
157 class PGOCounterPromoterHelper : public LoadAndStorePromoter {
158 public:
159  PGOCounterPromoterHelper(
161  BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
162  ArrayRef<Instruction *> InsertPts,
164  LoopInfo &LI)
165  : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
166  InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
167  assert(isa<LoadInst>(L));
168  assert(isa<StoreInst>(S));
169  SSA.AddAvailableValue(PH, Init);
170  }
171 
172  void doExtraRewritesBeforeFinalDeletion() override {
173  for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
174  BasicBlock *ExitBlock = ExitBlocks[i];
175  Instruction *InsertPos = InsertPts[i];
176  // Get LiveIn value into the ExitBlock. If there are multiple
177  // predecessors, the value is defined by a PHI node in this
178  // block.
179  Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
180  Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
181  Type *Ty = LiveInValue->getType();
182  IRBuilder<> Builder(InsertPos);
183  if (auto *AddrInst = dyn_cast_or_null<IntToPtrInst>(Addr)) {
184  // If isRuntimeCounterRelocationEnabled() is true then the address of
185  // the store instruction is computed with two instructions in
186  // InstrProfiling::getCounterAddress(). We need to copy those
187  // instructions to this block to compute Addr correctly.
188  // %BiasAdd = add i64 ptrtoint <__profc_>, <__llvm_profile_counter_bias>
189  // %Addr = inttoptr i64 %BiasAdd to i64*
190  auto *OrigBiasInst = dyn_cast<BinaryOperator>(AddrInst->getOperand(0));
191  assert(OrigBiasInst->getOpcode() == Instruction::BinaryOps::Add);
192  Value *BiasInst = Builder.Insert(OrigBiasInst->clone());
193  Addr = Builder.CreateIntToPtr(BiasInst, Ty->getPointerTo());
194  }
195  if (AtomicCounterUpdatePromoted)
196  // automic update currently can only be promoted across the current
197  // loop, not the whole loop nest.
198  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
199  MaybeAlign(),
201  else {
202  LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
203  auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
204  auto *NewStore = Builder.CreateStore(NewVal, Addr);
205 
206  // Now update the parent loop's candidate list:
207  if (IterativeCounterPromotion) {
208  auto *TargetLoop = LI.getLoopFor(ExitBlock);
209  if (TargetLoop)
210  LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
211  }
212  }
213  }
214  }
215 
216 private:
218  ArrayRef<BasicBlock *> ExitBlocks;
219  ArrayRef<Instruction *> InsertPts;
221  LoopInfo &LI;
222 };
223 
224 /// A helper class to do register promotion for all profile counter
225 /// updates in a loop.
226 ///
227 class PGOCounterPromoter {
228 public:
229  PGOCounterPromoter(
231  Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
232  : LoopToCandidates(LoopToCands), L(CurLoop), LI(LI), BFI(BFI) {
233 
234  // Skip collection of ExitBlocks and InsertPts for loops that will not be
235  // able to have counters promoted.
236  SmallVector<BasicBlock *, 8> LoopExitBlocks;
238 
239  L.getExitBlocks(LoopExitBlocks);
240  if (!isPromotionPossible(&L, LoopExitBlocks))
241  return;
242 
243  for (BasicBlock *ExitBlock : LoopExitBlocks) {
244  if (BlockSet.insert(ExitBlock).second) {
245  ExitBlocks.push_back(ExitBlock);
246  InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
247  }
248  }
249  }
250 
251  bool run(int64_t *NumPromoted) {
252  // Skip 'infinite' loops:
253  if (ExitBlocks.size() == 0)
254  return false;
255 
256  // Skip if any of the ExitBlocks contains a ret instruction.
257  // This is to prevent dumping of incomplete profile -- if the
258  // the loop is a long running loop and dump is called in the middle
259  // of the loop, the result profile is incomplete.
260  // FIXME: add other heuristics to detect long running loops.
261  if (SkipRetExitBlock) {
262  for (auto *BB : ExitBlocks)
263  if (isa<ReturnInst>(BB->getTerminator()))
264  return false;
265  }
266 
267  unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
268  if (MaxProm == 0)
269  return false;
270 
271  unsigned Promoted = 0;
272  for (auto &Cand : LoopToCandidates[&L]) {
273 
275  SSAUpdater SSA(&NewPHIs);
276  Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
277 
278  // If BFI is set, we will use it to guide the promotions.
279  if (BFI) {
280  auto *BB = Cand.first->getParent();
281  auto InstrCount = BFI->getBlockProfileCount(BB);
282  if (!InstrCount)
283  continue;
284  auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
285  // If the average loop trip count is not greater than 1.5, we skip
286  // promotion.
287  if (PreheaderCount && (*PreheaderCount * 3) >= (*InstrCount * 2))
288  continue;
289  }
290 
291  PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
292  L.getLoopPreheader(), ExitBlocks,
293  InsertPts, LoopToCandidates, LI);
294  Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
295  Promoted++;
296  if (Promoted >= MaxProm)
297  break;
298 
299  (*NumPromoted)++;
300  if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
301  break;
302  }
303 
304  LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
305  << L.getLoopDepth() << ")\n");
306  return Promoted != 0;
307  }
308 
309 private:
310  bool allowSpeculativeCounterPromotion(Loop *LP) {
311  SmallVector<BasicBlock *, 8> ExitingBlocks;
312  L.getExitingBlocks(ExitingBlocks);
313  // Not considierered speculative.
314  if (ExitingBlocks.size() == 1)
315  return true;
316  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
317  return false;
318  return true;
319  }
320 
321  // Check whether the loop satisfies the basic conditions needed to perform
322  // Counter Promotions.
323  bool
324  isPromotionPossible(Loop *LP,
325  const SmallVectorImpl<BasicBlock *> &LoopExitBlocks) {
326  // We can't insert into a catchswitch.
327  if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
328  return isa<CatchSwitchInst>(Exit->getTerminator());
329  }))
330  return false;
331 
332  if (!LP->hasDedicatedExits())
333  return false;
334 
335  BasicBlock *PH = LP->getLoopPreheader();
336  if (!PH)
337  return false;
338 
339  return true;
340  }
341 
342  // Returns the max number of Counter Promotions for LP.
343  unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
344  SmallVector<BasicBlock *, 8> LoopExitBlocks;
345  LP->getExitBlocks(LoopExitBlocks);
346  if (!isPromotionPossible(LP, LoopExitBlocks))
347  return 0;
348 
349  SmallVector<BasicBlock *, 8> ExitingBlocks;
350  LP->getExitingBlocks(ExitingBlocks);
351 
352  // If BFI is set, we do more aggressive promotions based on BFI.
353  if (BFI)
354  return (unsigned)-1;
355 
356  // Not considierered speculative.
357  if (ExitingBlocks.size() == 1)
358  return MaxNumOfPromotionsPerLoop;
359 
360  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
361  return 0;
362 
363  // Whether the target block is in a loop does not matter:
364  if (SpeculativeCounterPromotionToLoop)
365  return MaxNumOfPromotionsPerLoop;
366 
367  // Now check the target block:
368  unsigned MaxProm = MaxNumOfPromotionsPerLoop;
369  for (auto *TargetBlock : LoopExitBlocks) {
370  auto *TargetLoop = LI.getLoopFor(TargetBlock);
371  if (!TargetLoop)
372  continue;
373  unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
374  unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
375  MaxProm =
376  std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
377  PendingCandsInTarget);
378  }
379  return MaxProm;
380  }
381 
383  SmallVector<BasicBlock *, 8> ExitBlocks;
385  Loop &L;
386  LoopInfo &LI;
388 };
389 
390 enum class ValueProfilingCallType {
391  // Individual values are tracked. Currently used for indiret call target
392  // profiling.
393  Default,
394 
395  // MemOp: the memop size value profiling.
396  MemOp
397 };
398 
399 } // end anonymous namespace
400 
404  auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & {
406  };
407  if (!run(M, GetTLI))
408  return PreservedAnalyses::all();
409 
410  return PreservedAnalyses::none();
411 }
412 
413 bool InstrProfiling::lowerIntrinsics(Function *F) {
414  bool MadeChange = false;
415  PromotionCandidates.clear();
416  for (BasicBlock &BB : *F) {
417  for (Instruction &Instr : llvm::make_early_inc_range(BB)) {
418  if (auto *IPIS = dyn_cast<InstrProfIncrementInstStep>(&Instr)) {
419  lowerIncrement(IPIS);
420  MadeChange = true;
421  } else if (auto *IPI = dyn_cast<InstrProfIncrementInst>(&Instr)) {
422  lowerIncrement(IPI);
423  MadeChange = true;
424  } else if (auto *IPC = dyn_cast<InstrProfCoverInst>(&Instr)) {
425  lowerCover(IPC);
426  MadeChange = true;
427  } else if (auto *IPVP = dyn_cast<InstrProfValueProfileInst>(&Instr)) {
428  lowerValueProfileInst(IPVP);
429  MadeChange = true;
430  }
431  }
432  }
433 
434  if (!MadeChange)
435  return false;
436 
437  promoteCounterLoadStores(F);
438  return true;
439 }
440 
441 bool InstrProfiling::isRuntimeCounterRelocationEnabled() const {
442  // Mach-O don't support weak external references.
443  if (TT.isOSBinFormatMachO())
444  return false;
445 
446  if (RuntimeCounterRelocation.getNumOccurrences() > 0)
447  return RuntimeCounterRelocation;
448 
449  // Fuchsia uses runtime counter relocation by default.
450  return TT.isOSFuchsia();
451 }
452 
453 bool InstrProfiling::isCounterPromotionEnabled() const {
454  if (DoCounterPromotion.getNumOccurrences() > 0)
455  return DoCounterPromotion;
456 
457  return Options.DoCounterPromotion;
458 }
459 
460 void InstrProfiling::promoteCounterLoadStores(Function *F) {
461  if (!isCounterPromotionEnabled())
462  return;
463 
464  DominatorTree DT(*F);
465  LoopInfo LI(DT);
466  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
467 
468  std::unique_ptr<BlockFrequencyInfo> BFI;
469  if (Options.UseBFIInPromotion) {
470  std::unique_ptr<BranchProbabilityInfo> BPI;
471  BPI.reset(new BranchProbabilityInfo(*F, LI, &GetTLI(*F)));
472  BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
473  }
474 
475  for (const auto &LoadStore : PromotionCandidates) {
476  auto *CounterLoad = LoadStore.first;
477  auto *CounterStore = LoadStore.second;
478  BasicBlock *BB = CounterLoad->getParent();
479  Loop *ParentLoop = LI.getLoopFor(BB);
480  if (!ParentLoop)
481  continue;
482  LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
483  }
484 
486 
487  // Do a post-order traversal of the loops so that counter updates can be
488  // iteratively hoisted outside the loop nest.
489  for (auto *Loop : llvm::reverse(Loops)) {
490  PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
491  Promoter.run(&TotalCountersPromoted);
492  }
493 }
494 
495 static bool needsRuntimeHookUnconditionally(const Triple &TT) {
496  // On Fuchsia, we only need runtime hook if any counters are present.
497  if (TT.isOSFuchsia())
498  return false;
499 
500  return true;
501 }
502 
503 /// Check if the module contains uses of any profiling intrinsics.
505  auto containsIntrinsic = [&](int ID) {
506  if (auto *F = M.getFunction(Intrinsic::getName(ID)))
507  return !F->use_empty();
508  return false;
509  };
510  return containsIntrinsic(llvm::Intrinsic::instrprof_cover) ||
511  containsIntrinsic(llvm::Intrinsic::instrprof_increment) ||
512  containsIntrinsic(llvm::Intrinsic::instrprof_increment_step) ||
513  containsIntrinsic(llvm::Intrinsic::instrprof_value_profile);
514 }
515 
517  Module &M, std::function<const TargetLibraryInfo &(Function &F)> GetTLI) {
518  this->M = &M;
519  this->GetTLI = std::move(GetTLI);
520  NamesVar = nullptr;
521  NamesSize = 0;
522  ProfileDataMap.clear();
523  CompilerUsedVars.clear();
524  UsedVars.clear();
525  TT = Triple(M.getTargetTriple());
526 
527  bool MadeChange = false;
528  bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT);
529  if (NeedsRuntimeHook)
530  MadeChange = emitRuntimeHook();
531 
532  bool ContainsProfiling = containsProfilingIntrinsics(M);
533  GlobalVariable *CoverageNamesVar =
534  M.getNamedGlobal(getCoverageUnusedNamesVarName());
535  // Improve compile time by avoiding linear scans when there is no work.
536  if (!ContainsProfiling && !CoverageNamesVar)
537  return MadeChange;
538 
539  // We did not know how many value sites there would be inside
540  // the instrumented function. This is counting the number of instrumented
541  // target value sites to enter it as field in the profile data variable.
542  for (Function &F : M) {
543  InstrProfIncrementInst *FirstProfIncInst = nullptr;
544  for (BasicBlock &BB : F)
545  for (auto I = BB.begin(), E = BB.end(); I != E; I++)
546  if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
547  computeNumValueSiteCounts(Ind);
548  else if (FirstProfIncInst == nullptr)
549  FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
550 
551  // Value profiling intrinsic lowering requires per-function profile data
552  // variable to be created first.
553  if (FirstProfIncInst != nullptr)
554  static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
555  }
556 
557  for (Function &F : M)
558  MadeChange |= lowerIntrinsics(&F);
559 
560  if (CoverageNamesVar) {
561  lowerCoverageData(CoverageNamesVar);
562  MadeChange = true;
563  }
564 
565  if (!MadeChange)
566  return false;
567 
568  emitVNodes();
569  emitNameData();
570 
571  // Emit runtime hook for the cases where the target does not unconditionally
572  // require pulling in profile runtime, and coverage is enabled on code that is
573  // not eliminated by the front-end, e.g. unused functions with internal
574  // linkage.
575  if (!NeedsRuntimeHook && ContainsProfiling)
576  emitRuntimeHook();
577 
578  emitRegistration();
579  emitUses();
580  emitInitialization();
581  return true;
582 }
583 
585  Module &M, const TargetLibraryInfo &TLI,
586  ValueProfilingCallType CallType = ValueProfilingCallType::Default) {
587  LLVMContext &Ctx = M.getContext();
588  auto *ReturnTy = Type::getVoidTy(M.getContext());
589 
591  if (auto AK = TLI.getExtAttrForI32Param(false))
592  AL = AL.addParamAttribute(M.getContext(), 2, AK);
593 
595  CallType == ValueProfilingCallType::MemOp) &&
596  "Must be Default or MemOp");
597  Type *ParamTypes[] = {
598 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
600  };
601  auto *ValueProfilingCallTy =
602  FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
603  StringRef FuncName = CallType == ValueProfilingCallType::Default
606  return M.getOrInsertFunction(FuncName, ValueProfilingCallTy, AL);
607 }
608 
609 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
610  GlobalVariable *Name = Ind->getName();
612  uint64_t Index = Ind->getIndex()->getZExtValue();
613  auto &PD = ProfileDataMap[Name];
614  PD.NumValueSites[ValueKind] =
615  std::max(PD.NumValueSites[ValueKind], (uint32_t)(Index + 1));
616 }
617 
618 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
619  // TODO: Value profiling heavily depends on the data section which is omitted
620  // in lightweight mode. We need to move the value profile pointer to the
621  // Counter struct to get this working.
622  assert(
624  "Value profiling is not yet supported with lightweight instrumentation");
625  GlobalVariable *Name = Ind->getName();
626  auto It = ProfileDataMap.find(Name);
627  assert(It != ProfileDataMap.end() && It->second.DataVar &&
628  "value profiling detected in function with no counter incerement");
629 
630  GlobalVariable *DataVar = It->second.DataVar;
632  uint64_t Index = Ind->getIndex()->getZExtValue();
633  for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
634  Index += It->second.NumValueSites[Kind];
635 
636  IRBuilder<> Builder(Ind);
637  bool IsMemOpSize = (Ind->getValueKind()->getZExtValue() ==
638  llvm::InstrProfValueKind::IPVK_MemOPSize);
639  CallInst *Call = nullptr;
640  auto *TLI = &GetTLI(*Ind->getFunction());
641 
642  // To support value profiling calls within Windows exception handlers, funclet
643  // information contained within operand bundles needs to be copied over to
644  // the library call. This is required for the IR to be processed by the
645  // WinEHPrepare pass.
647  Ind->getOperandBundlesAsDefs(OpBundles);
648  if (!IsMemOpSize) {
649  Value *Args[3] = {Ind->getTargetValue(),
650  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
651  Builder.getInt32(Index)};
652  Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args,
653  OpBundles);
654  } else {
655  Value *Args[3] = {Ind->getTargetValue(),
656  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
657  Builder.getInt32(Index)};
658  Call = Builder.CreateCall(
659  getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp),
660  Args, OpBundles);
661  }
662  if (auto AK = TLI->getExtAttrForI32Param(false))
663  Call->addParamAttr(2, AK);
664  Ind->replaceAllUsesWith(Call);
665  Ind->eraseFromParent();
666 }
667 
668 Value *InstrProfiling::getCounterAddress(InstrProfInstBase *I) {
669  auto *Counters = getOrCreateRegionCounters(I);
671 
672  auto *Addr = Builder.CreateConstInBoundsGEP2_32(
673  Counters->getValueType(), Counters, 0, I->getIndex()->getZExtValue());
674 
675  if (!isRuntimeCounterRelocationEnabled())
676  return Addr;
677 
678  Type *Int64Ty = Type::getInt64Ty(M->getContext());
679  Function *Fn = I->getParent()->getParent();
680  LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn];
681  if (!BiasLI) {
682  IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front());
683  auto *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName());
684  if (!Bias) {
685  // Compiler must define this variable when runtime counter relocation
686  // is being used. Runtime has a weak external reference that is used
687  // to check whether that's the case or not.
688  Bias = new GlobalVariable(
689  *M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage,
691  Bias->setVisibility(GlobalVariable::HiddenVisibility);
692  // A definition that's weak (linkonce_odr) without being in a COMDAT
693  // section wouldn't lead to link errors, but it would lead to a dead
694  // data word from every TU but one. Putting it in COMDAT ensures there
695  // will be exactly one data slot in the link.
696  if (TT.supportsCOMDAT())
697  Bias->setComdat(M->getOrInsertComdat(Bias->getName()));
698  }
699  BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias);
700  }
701  auto *Add = Builder.CreateAdd(Builder.CreatePtrToInt(Addr, Int64Ty), BiasLI);
702  return Builder.CreateIntToPtr(Add, Addr->getType());
703 }
704 
705 void InstrProfiling::lowerCover(InstrProfCoverInst *CoverInstruction) {
706  auto *Addr = getCounterAddress(CoverInstruction);
707  IRBuilder<> Builder(CoverInstruction);
708  // We store zero to represent that this block is covered.
709  Builder.CreateStore(Builder.getInt8(0), Addr);
710  CoverInstruction->eraseFromParent();
711 }
712 
713 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
714  auto *Addr = getCounterAddress(Inc);
715 
716  IRBuilder<> Builder(Inc);
717  if (Options.Atomic || AtomicCounterUpdateAll ||
718  (Inc->getIndex()->isZeroValue() && AtomicFirstCounter)) {
719  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
721  } else {
722  Value *IncStep = Inc->getStep();
723  Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
724  auto *Count = Builder.CreateAdd(Load, Inc->getStep());
725  auto *Store = Builder.CreateStore(Count, Addr);
726  if (isCounterPromotionEnabled())
727  PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
728  }
729  Inc->eraseFromParent();
730 }
731 
732 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
733  ConstantArray *Names =
734  cast<ConstantArray>(CoverageNamesVar->getInitializer());
735  for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
736  Constant *NC = Names->getOperand(I);
737  Value *V = NC->stripPointerCasts();
738  assert(isa<GlobalVariable>(V) && "Missing reference to function name");
739  GlobalVariable *Name = cast<GlobalVariable>(V);
740 
741  Name->setLinkage(GlobalValue::PrivateLinkage);
742  ReferencedNames.push_back(Name);
743  if (isa<ConstantExpr>(NC))
744  NC->dropAllReferences();
745  }
746  CoverageNamesVar->eraseFromParent();
747 }
748 
749 /// Get the name of a profiling variable for a particular function.
750 static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix,
751  bool &Renamed) {
752  StringRef NamePrefix = getInstrProfNameVarPrefix();
753  StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
754  Function *F = Inc->getParent()->getParent();
755  Module *M = F->getParent();
756  if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
757  !canRenameComdatFunc(*F)) {
758  Renamed = false;
759  return (Prefix + Name).str();
760  }
761  Renamed = true;
762  uint64_t FuncHash = Inc->getHash()->getZExtValue();
763  SmallVector<char, 24> HashPostfix;
764  if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
765  return (Prefix + Name).str();
766  return (Prefix + Name + "." + Twine(FuncHash)).str();
767 }
768 
770  auto *MD = dyn_cast_or_null<ConstantAsMetadata>(M.getModuleFlag(Flag));
771  if (!MD)
772  return 0;
773 
774  // If the flag is a ConstantAsMetadata, it should be an integer representable
775  // in 64-bits.
776  return cast<ConstantInt>(MD->getValue())->getZExtValue();
777 }
778 
779 static bool enablesValueProfiling(const Module &M) {
780  return isIRPGOFlagSet(&M) ||
781  getIntModuleFlagOrZero(M, "EnableValueProfiling") != 0;
782 }
783 
784 // Conservatively returns true if data variables may be referenced by code.
785 static bool profDataReferencedByCode(const Module &M) {
786  return enablesValueProfiling(M);
787 }
788 
789 static inline bool shouldRecordFunctionAddr(Function *F) {
790  // Only record function addresses if IR PGO is enabled or if clang value
791  // profiling is enabled. Recording function addresses greatly increases object
792  // file size, because it prevents the inliner from deleting functions that
793  // have been inlined everywhere.
794  if (!profDataReferencedByCode(*F->getParent()))
795  return false;
796 
797  // Check the linkage
798  bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
799  if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
800  !HasAvailableExternallyLinkage)
801  return true;
802 
803  // A function marked 'alwaysinline' with available_externally linkage can't
804  // have its address taken. Doing so would create an undefined external ref to
805  // the function, which would fail to link.
806  if (HasAvailableExternallyLinkage &&
807  F->hasFnAttribute(Attribute::AlwaysInline))
808  return false;
809 
810  // Prohibit function address recording if the function is both internal and
811  // COMDAT. This avoids the profile data variable referencing internal symbols
812  // in COMDAT.
813  if (F->hasLocalLinkage() && F->hasComdat())
814  return false;
815 
816  // Check uses of this function for other than direct calls or invokes to it.
817  // Inline virtual functions have linkeOnceODR linkage. When a key method
818  // exists, the vtable will only be emitted in the TU where the key method
819  // is defined. In a TU where vtable is not available, the function won't
820  // be 'addresstaken'. If its address is not recorded here, the profile data
821  // with missing address may be picked by the linker leading to missing
822  // indirect call target info.
823  return F->hasAddressTaken() || F->hasLinkOnceLinkage();
824 }
825 
827  // Don't do this for Darwin. compiler-rt uses linker magic.
828  if (TT.isOSDarwin())
829  return false;
830  // Use linker script magic to get data/cnts/name start/end.
831  if (TT.isOSAIX() || TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
832  TT.isOSSolaris() || TT.isOSFuchsia() || TT.isPS() || TT.isOSWindows())
833  return false;
834 
835  return true;
836 }
837 
839 InstrProfiling::createRegionCounters(InstrProfInstBase *Inc, StringRef Name,
840  GlobalValue::LinkageTypes Linkage) {
841  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
842  auto &Ctx = M->getContext();
843  GlobalVariable *GV;
844  if (isa<InstrProfCoverInst>(Inc)) {
845  auto *CounterTy = Type::getInt8Ty(Ctx);
846  auto *CounterArrTy = ArrayType::get(CounterTy, NumCounters);
847  // TODO: `Constant::getAllOnesValue()` does not yet accept an array type.
848  std::vector<Constant *> InitialValues(NumCounters,
849  Constant::getAllOnesValue(CounterTy));
850  GV = new GlobalVariable(*M, CounterArrTy, false, Linkage,
851  ConstantArray::get(CounterArrTy, InitialValues),
852  Name);
853  GV->setAlignment(Align(1));
854  } else {
855  auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
856  GV = new GlobalVariable(*M, CounterTy, false, Linkage,
857  Constant::getNullValue(CounterTy), Name);
858  GV->setAlignment(Align(8));
859  }
860  return GV;
861 }
862 
864 InstrProfiling::getOrCreateRegionCounters(InstrProfInstBase *Inc) {
865  GlobalVariable *NamePtr = Inc->getName();
866  auto &PD = ProfileDataMap[NamePtr];
867  if (PD.RegionCounters)
868  return PD.RegionCounters;
869 
870  // Match the linkage and visibility of the name global.
871  Function *Fn = Inc->getParent()->getParent();
873  GlobalValue::VisibilityTypes Visibility = NamePtr->getVisibility();
874 
875  // Use internal rather than private linkage so the counter variable shows up
876  // in the symbol table when using debug info for correlation.
878  Linkage == GlobalValue::PrivateLinkage)
880 
881  // Due to the limitation of binder as of 2021/09/28, the duplicate weak
882  // symbols in the same csect won't be discarded. When there are duplicate weak
883  // symbols, we can NOT guarantee that the relocations get resolved to the
884  // intended weak symbol, so we can not ensure the correctness of the relative
885  // CounterPtr, so we have to use private linkage for counter and data symbols.
886  if (TT.isOSBinFormatXCOFF()) {
888  Visibility = GlobalValue::DefaultVisibility;
889  }
890  // Move the name variable to the right section. Place them in a COMDAT group
891  // if the associated function is a COMDAT. This will make sure that only one
892  // copy of counters of the COMDAT function will be emitted after linking. Keep
893  // in mind that this pass may run before the inliner, so we need to create a
894  // new comdat group for the counters and profiling data. If we use the comdat
895  // of the parent function, that will result in relocations against discarded
896  // sections.
897  //
898  // If the data variable is referenced by code, counters and data have to be
899  // in different comdats for COFF because the Visual C++ linker will report
900  // duplicate symbol errors if there are multiple external symbols with the
901  // same name marked IMAGE_COMDAT_SELECT_ASSOCIATIVE.
902  //
903  // For ELF, when not using COMDAT, put counters, data and values into a
904  // nodeduplicate COMDAT which is lowered to a zero-flag section group. This
905  // allows -z start-stop-gc to discard the entire group when the function is
906  // discarded.
907  bool DataReferencedByCode = profDataReferencedByCode(*M);
908  bool NeedComdat = needsComdatForCounter(*Fn, *M);
909  bool Renamed;
910  std::string CntsVarName =
911  getVarName(Inc, getInstrProfCountersVarPrefix(), Renamed);
912  std::string DataVarName =
913  getVarName(Inc, getInstrProfDataVarPrefix(), Renamed);
914  auto MaybeSetComdat = [&](GlobalVariable *GV) {
915  bool UseComdat = (NeedComdat || TT.isOSBinFormatELF());
916  if (UseComdat) {
917  StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode
918  ? GV->getName()
919  : CntsVarName;
920  Comdat *C = M->getOrInsertComdat(GroupName);
921  if (!NeedComdat)
922  C->setSelectionKind(Comdat::NoDeduplicate);
923  GV->setComdat(C);
924  // COFF doesn't allow the comdat group leader to have private linkage, so
925  // upgrade private linkage to internal linkage to produce a symbol table
926  // entry.
927  if (TT.isOSBinFormatCOFF() && GV->hasPrivateLinkage())
929  }
930  };
931 
932  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
933  LLVMContext &Ctx = M->getContext();
934 
935  auto *CounterPtr = createRegionCounters(Inc, CntsVarName, Linkage);
936  CounterPtr->setVisibility(Visibility);
937  CounterPtr->setSection(
938  getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
939  CounterPtr->setLinkage(Linkage);
940  MaybeSetComdat(CounterPtr);
941  PD.RegionCounters = CounterPtr;
942  if (DebugInfoCorrelate) {
943  if (auto *SP = Fn->getSubprogram()) {
944  DIBuilder DB(*M, true, SP->getUnit());
945  Metadata *FunctionNameAnnotation[] = {
948  };
949  Metadata *CFGHashAnnotation[] = {
952  };
953  Metadata *NumCountersAnnotation[] = {
956  };
957  auto Annotations = DB.getOrCreateArray({
958  MDNode::get(Ctx, FunctionNameAnnotation),
959  MDNode::get(Ctx, CFGHashAnnotation),
960  MDNode::get(Ctx, NumCountersAnnotation),
961  });
962  auto *DICounter = DB.createGlobalVariableExpression(
963  SP, CounterPtr->getName(), /*LinkageName=*/StringRef(), SP->getFile(),
964  /*LineNo=*/0, DB.createUnspecifiedType("Profile Data Type"),
965  CounterPtr->hasLocalLinkage(), /*IsDefined=*/true, /*Expr=*/nullptr,
966  /*Decl=*/nullptr, /*TemplateParams=*/nullptr, /*AlignInBits=*/0,
967  Annotations);
968  CounterPtr->addDebugInfo(DICounter);
969  DB.finalize();
970  } else {
971  std::string Msg = ("Missing debug info for function " + Fn->getName() +
972  "; required for profile correlation.")
973  .str();
974  Ctx.diagnose(
975  DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
976  }
977  }
978 
979  auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
980  // Allocate statically the array of pointers to value profile nodes for
981  // the current function.
982  Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
983  uint64_t NS = 0;
984  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
985  NS += PD.NumValueSites[Kind];
986  if (NS > 0 && ValueProfileStaticAlloc &&
988  ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
989  auto *ValuesVar = new GlobalVariable(
990  *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy),
991  getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed));
992  ValuesVar->setVisibility(Visibility);
993  ValuesVar->setSection(
994  getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
995  ValuesVar->setAlignment(Align(8));
996  MaybeSetComdat(ValuesVar);
997  ValuesPtrExpr =
999  }
1000 
1001  if (DebugInfoCorrelate) {
1002  // Mark the counter variable as used so that it isn't optimized out.
1003  CompilerUsedVars.push_back(PD.RegionCounters);
1004  return PD.RegionCounters;
1005  }
1006 
1007  // Create data variable.
1008  auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext());
1009  auto *Int16Ty = Type::getInt16Ty(Ctx);
1010  auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
1011  Type *DataTypes[] = {
1012 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
1014  };
1015  auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
1016 
1017  Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
1018  ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
1019  : ConstantPointerNull::get(Int8PtrTy);
1020 
1021  Constant *Int16ArrayVals[IPVK_Last + 1];
1022  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1023  Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
1024 
1025  // If the data variable is not referenced by code (if we don't emit
1026  // @llvm.instrprof.value.profile, NS will be 0), and the counter keeps the
1027  // data variable live under linker GC, the data variable can be private. This
1028  // optimization applies to ELF.
1029  //
1030  // On COFF, a comdat leader cannot be local so we require DataReferencedByCode
1031  // to be false.
1032  //
1033  // If profd is in a deduplicate comdat, NS==0 with a hash suffix guarantees
1034  // that other copies must have the same CFG and cannot have value profiling.
1035  // If no hash suffix, other profd copies may be referenced by code.
1036  if (NS == 0 && !(DataReferencedByCode && NeedComdat && !Renamed) &&
1037  (TT.isOSBinFormatELF() ||
1038  (!DataReferencedByCode && TT.isOSBinFormatCOFF()))) {
1040  Visibility = GlobalValue::DefaultVisibility;
1041  }
1042  auto *Data =
1043  new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName);
1044  // Reference the counter variable with a label difference (link-time
1045  // constant).
1046  auto *RelativeCounterPtr =
1047  ConstantExpr::getSub(ConstantExpr::getPtrToInt(CounterPtr, IntPtrTy),
1048  ConstantExpr::getPtrToInt(Data, IntPtrTy));
1049 
1050  Constant *DataVals[] = {
1051 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
1053  };
1054  Data->setInitializer(ConstantStruct::get(DataTy, DataVals));
1055 
1056  Data->setVisibility(Visibility);
1057  Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
1058  Data->setAlignment(Align(INSTR_PROF_DATA_ALIGNMENT));
1059  MaybeSetComdat(Data);
1060 
1061  PD.DataVar = Data;
1062 
1063  // Mark the data variable as used so that it isn't stripped out.
1064  CompilerUsedVars.push_back(Data);
1065  // Now that the linkage set by the FE has been passed to the data and counter
1066  // variables, reset Name variable's linkage and visibility to private so that
1067  // it can be removed later by the compiler.
1069  // Collect the referenced names to be used by emitNameData.
1070  ReferencedNames.push_back(NamePtr);
1071 
1072  return PD.RegionCounters;
1073 }
1074 
1075 void InstrProfiling::emitVNodes() {
1076  if (!ValueProfileStaticAlloc)
1077  return;
1078 
1079  // For now only support this on platforms that do
1080  // not require runtime registration to discover
1081  // named section start/end.
1083  return;
1084 
1085  size_t TotalNS = 0;
1086  for (auto &PD : ProfileDataMap) {
1087  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
1088  TotalNS += PD.second.NumValueSites[Kind];
1089  }
1090 
1091  if (!TotalNS)
1092  return;
1093 
1094  uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
1095 // Heuristic for small programs with very few total value sites.
1096 // The default value of vp-counters-per-site is chosen based on
1097 // the observation that large apps usually have a low percentage
1098 // of value sites that actually have any profile data, and thus
1099 // the average number of counters per site is low. For small
1100 // apps with very few sites, this may not be true. Bump up the
1101 // number of counters in this case.
1102 #define INSTR_PROF_MIN_VAL_COUNTS 10
1103  if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
1104  NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
1105 
1106  auto &Ctx = M->getContext();
1107  Type *VNodeTypes[] = {
1108 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
1110  };
1111  auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
1112 
1113  ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
1114  auto *VNodesVar = new GlobalVariable(
1115  *M, VNodesTy, false, GlobalValue::PrivateLinkage,
1117  VNodesVar->setSection(
1118  getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
1119  // VNodesVar is used by runtime but not referenced via relocation by other
1120  // sections. Conservatively make it linker retained.
1121  UsedVars.push_back(VNodesVar);
1122 }
1123 
1124 void InstrProfiling::emitNameData() {
1125  std::string UncompressedData;
1126 
1127  if (ReferencedNames.empty())
1128  return;
1129 
1130  std::string CompressedNameStr;
1131  if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
1134  }
1135 
1136  auto &Ctx = M->getContext();
1137  auto *NamesVal =
1138  ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false);
1139  NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
1140  GlobalValue::PrivateLinkage, NamesVal,
1142  NamesSize = CompressedNameStr.size();
1143  NamesVar->setSection(
1144  getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
1145  // On COFF, it's important to reduce the alignment down to 1 to prevent the
1146  // linker from inserting padding before the start of the names section or
1147  // between names entries.
1148  NamesVar->setAlignment(Align(1));
1149  // NamesVar is used by runtime but not referenced via relocation by other
1150  // sections. Conservatively make it linker retained.
1151  UsedVars.push_back(NamesVar);
1152 
1153  for (auto *NamePtr : ReferencedNames)
1154  NamePtr->eraseFromParent();
1155 }
1156 
1157 void InstrProfiling::emitRegistration() {
1159  return;
1160 
1161  // Construct the function.
1162  auto *VoidTy = Type::getVoidTy(M->getContext());
1163  auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
1164  auto *Int64Ty = Type::getInt64Ty(M->getContext());
1165  auto *RegisterFTy = FunctionType::get(VoidTy, false);
1166  auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
1168  RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1169  if (Options.NoRedZone)
1170  RegisterF->addFnAttr(Attribute::NoRedZone);
1171 
1172  auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
1173  auto *RuntimeRegisterF =
1176 
1177  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
1178  for (Value *Data : CompilerUsedVars)
1179  if (!isa<Function>(Data))
1180  IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1181  for (Value *Data : UsedVars)
1182  if (Data != NamesVar && !isa<Function>(Data))
1183  IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
1184 
1185  if (NamesVar) {
1186  Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
1187  auto *NamesRegisterTy =
1188  FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
1189  auto *NamesRegisterF =
1192  IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
1193  IRB.getInt64(NamesSize)});
1194  }
1195 
1196  IRB.CreateRetVoid();
1197 }
1198 
1199 bool InstrProfiling::emitRuntimeHook() {
1200  // We expect the linker to be invoked with -u<hook_var> flag for Linux
1201  // in which case there is no need to emit the external variable.
1202  if (TT.isOSLinux() || TT.isOSAIX())
1203  return false;
1204 
1205  // If the module's provided its own runtime, we don't need to do anything.
1206  if (M->getGlobalVariable(getInstrProfRuntimeHookVarName()))
1207  return false;
1208 
1209  // Declare an external variable that will pull in the runtime initialization.
1210  auto *Int32Ty = Type::getInt32Ty(M->getContext());
1211  auto *Var =
1213  nullptr, getInstrProfRuntimeHookVarName());
1214  Var->setVisibility(GlobalValue::HiddenVisibility);
1215 
1216  if (TT.isOSBinFormatELF() && !TT.isPS()) {
1217  // Mark the user variable as used so that it isn't stripped out.
1218  CompilerUsedVars.push_back(Var);
1219  } else {
1220  // Make a function that uses it.
1224  User->addFnAttr(Attribute::NoInline);
1225  if (Options.NoRedZone)
1226  User->addFnAttr(Attribute::NoRedZone);
1227  User->setVisibility(GlobalValue::HiddenVisibility);
1228  if (TT.supportsCOMDAT())
1229  User->setComdat(M->getOrInsertComdat(User->getName()));
1230 
1231  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User));
1232  auto *Load = IRB.CreateLoad(Int32Ty, Var);
1233  IRB.CreateRet(Load);
1234 
1235  // Mark the function as used so that it isn't stripped out.
1236  CompilerUsedVars.push_back(User);
1237  }
1238  return true;
1239 }
1240 
1241 void InstrProfiling::emitUses() {
1242  // The metadata sections are parallel arrays. Optimizers (e.g.
1243  // GlobalOpt/ConstantMerge) may not discard associated sections as a unit, so
1244  // we conservatively retain all unconditionally in the compiler.
1245  //
1246  // On ELF and Mach-O, the linker can guarantee the associated sections will be
1247  // retained or discarded as a unit, so llvm.compiler.used is sufficient.
1248  // Similarly on COFF, if prof data is not referenced by code we use one comdat
1249  // and ensure this GC property as well. Otherwise, we have to conservatively
1250  // make all of the sections retained by the linker.
1251  if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() ||
1253  appendToCompilerUsed(*M, CompilerUsedVars);
1254  else
1255  appendToUsed(*M, CompilerUsedVars);
1256 
1257  // We do not add proper references from used metadata sections to NamesVar and
1258  // VNodesVar, so we have to be conservative and place them in llvm.used
1259  // regardless of the target,
1260  appendToUsed(*M, UsedVars);
1261 }
1262 
1263 void InstrProfiling::emitInitialization() {
1264  // Create ProfileFileName variable. Don't don't this for the
1265  // context-sensitive instrumentation lowering: This lowering is after
1266  // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1267  // have already create the variable before LTO/ThinLTO linking.
1268  if (!IsCS)
1270  Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1271  if (!RegisterF)
1272  return;
1273 
1274  // Create the initialization function.
1275  auto *VoidTy = Type::getVoidTy(M->getContext());
1276  auto *F = Function::Create(FunctionType::get(VoidTy, false),
1279  F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1280  F->addFnAttr(Attribute::NoInline);
1281  if (Options.NoRedZone)
1282  F->addFnAttr(Attribute::NoRedZone);
1283 
1284  // Add the basic block and the necessary calls.
1285  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1286  IRB.CreateCall(RegisterF, {});
1287  IRB.CreateRetVoid();
1288 
1289  appendToGlobalCtors(*M, F, 0);
1290 }
INSTR_PROF_MIN_VAL_COUNTS
#define INSTR_PROF_MIN_VAL_COUNTS
i
i
Definition: README.txt:29
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
Int32Ty
IntegerType * Int32Ty
Definition: NVVMIntrRange.cpp:67
llvm::GlobalVariable::eraseFromParent
void eraseFromParent()
eraseFromParent - This method unlinks 'this' from the containing module and deletes it.
Definition: Globals.cpp:464
llvm::GlobalObject::setComdat
void setComdat(Comdat *C)
Definition: Globals.cpp:194
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::CallBase::getOperandBundlesAsDefs
void getOperandBundlesAsDefs(SmallVectorImpl< OperandBundleDef > &Defs) const
Return the list of operand bundles attached to this instruction as a vector of OperandBundleDefs.
Definition: Instructions.cpp:416
llvm::InstrProfIncrementInst::getStep
Value * getStep() const
Definition: IntrinsicInst.cpp:268
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::AArch64CC::AL
@ AL
Definition: AArch64BaseInfo.h:269
llvm::LoopBase::getExitBlocks
void getExitBlocks(SmallVectorImpl< BlockT * > &ExitBlocks) const
Return all of the successor blocks of this loop.
Definition: LoopInfoImpl.h:64
needsRuntimeHookUnconditionally
static bool needsRuntimeHookUnconditionally(const Triple &TT)
Definition: InstrProfiling.cpp:495
llvm::StructType::get
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:406
llvm::Type::getInt8PtrTy
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:291
llvm::getInstrProfNamesVarName
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
Definition: InstrProf.h:106
getVarName
static std::string getVarName(InstrProfInstBase *Inc, StringRef Prefix, bool &Renamed)
Get the name of a profiling variable for a particular function.
Definition: InstrProfiling.cpp:750
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:104
IntrinsicInst.h
llvm::DIBuilder
Definition: DIBuilder.h:42
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:774
llvm::Triple::isOSBinFormatCOFF
bool isOSBinFormatCOFF() const
Tests whether the OS uses the COFF binary format.
Definition: Triple.h:678
llvm::cl::Prefix
@ Prefix
Definition: CommandLine.h:161
llvm::InstrProfCorrelator::NumCountersAttributeName
static const char * NumCountersAttributeName
Definition: InstrProfCorrelator.h:59
llvm::LoadAndStorePromoter
Helper class for promoting a collection of loads and stores into SSA Form using the SSAUpdater.
Definition: SSAUpdater.h:136
llvm::GlobalValue::getLinkage
LinkageTypes getLinkage() const
Definition: GlobalValue.h:539
llvm::GlobalValue::HiddenVisibility
@ HiddenVisibility
The GV is hidden.
Definition: GlobalValue.h:64
llvm::InstrProfOptions::UseBFIInPromotion
bool UseBFIInPromotion
Definition: Instrumentation.h:117
llvm::Function
Definition: Function.h:60
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:547
StringRef.h
llvm::ConstantStruct::get
static Constant * get(StructType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1306
llvm::NVPTX::LoadStore
LoadStore
Definition: NVPTX.h:99
Pass.h
llvm::InstrProfInstBase
A base class for all instrprof intrinsics.
Definition: IntrinsicInst.h:1319
Loops
Hexagon Hardware Loops
Definition: HexagonHardwareLoops.cpp:374
llvm::Triple::isPS
bool isPS() const
Tests whether the target is the PS4 or PS5 platform.
Definition: Triple.h:720
llvm::needsComdatForCounter
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:1134
llvm::Comdat::NoDeduplicate
@ NoDeduplicate
No deduplication is performed.
Definition: Comdat.h:39
llvm::getPGOFuncNameVarInitializer
StringRef getPGOFuncNameVarInitializer(GlobalVariable *NameVar)
Return the initializer in string of the PGO name var NameVar.
Definition: InstrProf.cpp:476
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::Function::getSubprogram
DISubprogram * getSubprogram() const
Get the attached subprogram.
Definition: Metadata.cpp:1626
ErrorHandling.h
llvm::Function::getEntryBlock
const BasicBlock & getEntryBlock() const
Definition: Function.h:691
llvm::Intrinsic::getName
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:942
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:39
llvm::ConstantExpr::getBitCast
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2202
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:361
Error.h
llvm::AtomicOrdering::SequentiallyConsistent
@ SequentiallyConsistent
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
FAM
FunctionAnalysisManager FAM
Definition: PassBuilderBindings.cpp:59
llvm::getInstrProfRuntimeHookVarName
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:154
llvm::MemOp
Definition: TargetLowering.h:111
InstrCount
static unsigned InstrCount
Definition: DFAPacketizer.cpp:53
llvm::PreservedAnalyses::none
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
containsProfilingIntrinsics
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
Definition: InstrProfiling.cpp:504
Module.h
llvm::AttributeList
Definition: Attributes.h:430
llvm::TargetLibraryInfo::getExtAttrForI32Param
Attribute::AttrKind getExtAttrForI32Param(bool Signed=true) const
Returns extension attribute kind to be used for i32 parameters corresponding to C-level int or unsign...
Definition: TargetLibraryInfo.h:390
llvm::InstrProfOptions::Atomic
bool Atomic
Definition: Instrumentation.h:114
llvm::getInstrProfNamesRegFuncName
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:141
llvm::InstrProfInstBase::getIndex
ConstantInt * getIndex() const
Definition: IntrinsicInst.cpp:262
llvm::InstrProfCorrelator::CFGHashAttributeName
static const char * CFGHashAttributeName
Definition: InstrProfCorrelator.h:58
llvm::Triple::isOSBinFormatXCOFF
bool isOSBinFormatXCOFF() const
Tests whether the OS uses the XCOFF binary format.
Definition: Triple.h:696
llvm::GlobalValue::UnnamedAddr::Global
@ Global
llvm::InstrProfInstBase::getNumCounters
ConstantInt * getNumCounters() const
Definition: IntrinsicInst.cpp:256
llvm::StringRef::substr
StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:564
llvm::GlobalValue::DefaultVisibility
@ DefaultVisibility
The GV is visible.
Definition: GlobalValue.h:63
llvm::ConstantAsMetadata::get
static ConstantAsMetadata * get(Constant *C)
Definition: Metadata.h:420
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
llvm::Triple::isOSLinux
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:636
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::max
Expected< ExpressionValue > max(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:337
llvm::InstrProfInstBase::getHash
ConstantInt * getHash() const
Definition: IntrinsicInst.h:1327
llvm::GlobalValue::LinkageTypes
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
llvm::ArrayType
Class to represent array types.
Definition: DerivedTypes.h:357
llvm::createProfileFileNameVar
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1205
llvm::Type::getInt8Ty
static IntegerType * getInt8Ty(LLVMContext &C)
Definition: Type.cpp:237
llvm::Data
@ Data
Definition: SIMachineScheduler.h:55
llvm::Type::getInt32Ty
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:239
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::MDNode::get
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
Definition: Metadata.h:1400
F
#define F(x, y, z)
Definition: MD5.cpp:55
InstrProfData.inc
llvm::AtomicOrdering::Monotonic
@ Monotonic
getIntModuleFlagOrZero
static uint64_t getIntModuleFlagOrZero(const Module &M, StringRef Flag)
Definition: InstrProfiling.cpp:769
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::Annotations
Annotations lets you mark points and ranges inside source code, for tests:
Definition: Annotations.h:54
llvm::AArch64PACKey::DB
@ DB
Definition: AArch64BaseInfo.h:822
llvm::InstrProfValueProfileInst::getValueKind
ConstantInt * getValueKind() const
Definition: IntrinsicInst.h:1384
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ConstantExpr::getSub
static Constant * getSub(Constant *C1, Constant *C2, bool HasNUW=false, bool HasNSW=false)
Definition: Constants.cpp:2637
llvm::ConstantPointerNull
A constant pointer value that points to null.
Definition: Constants.h:535
Instruction.h
CommandLine.h
llvm::AMDGPU::HSAMD::ValueKind
ValueKind
Value kinds.
Definition: AMDGPUMetadata.h:78
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition: BlockFrequencyInfo.h:37
llvm::Triple::isOSBinFormatELF
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Definition: Triple.h:673
GlobalValue.h
llvm::GlobalValue::VisibilityTypes
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:62
Constants.h
InstrProfiling.h
llvm::X86II::PD
@ PD
Definition: X86BaseInfo.h:792
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::getInstrProfValuesVarPrefix
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:99
llvm::DS_Warning
@ DS_Warning
Definition: DiagnosticInfo.h:51
llvm::GlobalObject::setSection
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:248
llvm::User
Definition: User.h:44
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::InstrProfCorrelator::FunctionNameAttributeName
static const char * FunctionNameAttributeName
Definition: InstrProfCorrelator.h:57
Twine.h
llvm::InstrProfValueProfileInst::getTargetValue
Value * getTargetValue() const
Definition: IntrinsicInst.h:1380
llvm::BranchProbabilityInfo
Analysis providing branch probability information.
Definition: BranchProbabilityInfo.h:113
llvm::ConstantExpr::getPtrToInt
static Constant * getPtrToInt(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:2174
llvm::canRenameComdatFunc
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:1182
llvm::GlobalValue::hasPrivateLinkage
bool hasPrivateLinkage() const
Definition: GlobalValue.h:520
TargetLibraryInfo.h
llvm::getInstrProfCountersVarPrefix
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:96
llvm::Triple::supportsCOMDAT
bool supportsCOMDAT() const
Tests whether the target supports comdat.
Definition: Triple.h:942
llvm::BasicBlock::getFirstInsertionPt
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:246
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::MaybeAlign
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition: Alignment.h:117
llvm::dwarf::toStringRef
StringRef toStringRef(const Optional< DWARFFormValue > &V, StringRef Default={})
Take an optional DWARFFormValue and try to extract a string value from it.
Definition: DWARFFormValue.h:193
llvm::getInstrProfInitFuncName
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:149
llvm::ConstantArray
ConstantArray - Constant Array Declarations.
Definition: Constants.h:410
llvm::Constant::getAllOnesValue
static Constant * getAllOnesValue(Type *Ty)
Definition: Constants.cpp:395
llvm::Triple::isOSBinFormatMachO
bool isOSBinFormatMachO() const
Tests whether the environment is MachO.
Definition: Triple.h:686
llvm::Instruction
Definition: Instruction.h:42
llvm::MCID::Flag
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:147
InstrProf.h
llvm::Triple::getObjectFormat
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
Definition: Triple.h:380
llvm::appendToCompilerUsed
void appendToCompilerUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.compiler.used list.
Definition: ModuleUtils.cpp:109
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
llvm::LoopBase::getExitingBlocks
void getExitingBlocks(SmallVectorImpl< BlockT * > &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
Definition: LoopInfoImpl.h:33
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
llvm::InstrProfValueProfileInst::getIndex
ConstantInt * getIndex() const
Definition: IntrinsicInst.h:1389
Align
uint64_t Align
Definition: ELFObjHandler.cpp:81
llvm::InstrProfOptions::DoCounterPromotion
bool DoCounterPromotion
Definition: Instrumentation.h:111
llvm::GlobalValue::InternalLinkage
@ InternalLinkage
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
llvm::Comdat
Definition: Comdat.h:33
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:155
llvm::Metadata
Root of the metadata hierarchy.
Definition: Metadata.h:62
llvm::Triple::isOSAIX
bool isOSAIX() const
Tests whether the OS is AIX.
Definition: Triple.h:668
llvm::AtomicRMWInst::Add
@ Add
*p = old + v
Definition: Instructions.h:730
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
Type.h
llvm::dxil::PointerTypeAnalysis::run
PointerTypeMap run(const Module &M)
Compute the PointerTypeMap for the module M.
Definition: PointerTypeAnalysis.cpp:189
LoopInfo.h
llvm::Triple::isOSFuchsia
bool isOSFuchsia() const
Definition: Triple.h:547
llvm::getInstrProfVNodesVarName
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:102
llvm::getInstrProfNameVarPrefix
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:90
BasicBlock.h
llvm::cl::opt< bool >
SSA
Memory SSA
Definition: MemorySSA.cpp:73
llvm::GlobalVariable::getInitializer
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
Definition: GlobalVariable.h:135
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:81
BranchProbabilityInfo.h
llvm::InstrProfValueProfileInst
This represents the llvm.instrprof.value.profile intrinsic.
Definition: IntrinsicInst.h:1371
uint64_t
llvm::GlobalValue::getVisibility
VisibilityTypes getVisibility() const
Definition: GlobalValue.h:242
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:78
llvm::ConstantPointerNull::get
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1699
move
compiles ldr LCPI1_0 ldr ldr mov lsr tst moveq r1 ldr LCPI1_1 and r0 bx lr It would be better to do something like to fold the shift into the conditional move
Definition: README.txt:546
llvm::InstrProfOptions::InstrProfileOutput
std::string InstrProfileOutput
Definition: Instrumentation.h:120
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
profDataReferencedByCode
static bool profDataReferencedByCode(const Module &M)
Definition: InstrProfiling.cpp:785
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:53
llvm::DenseMap
Definition: DenseMap.h:714
shouldRecordFunctionAddr
static bool shouldRecordFunctionAddr(Function *F)
Definition: InstrProfiling.cpp:789
llvm::isIRPGOFlagSet
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:1160
llvm::LoopInfoBase::getLoopFor
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:992
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:447
llvm::appendToUsed
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
Definition: ModuleUtils.cpp:105
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:716
DIBuilder.h
llvm::LoopBase::getLoopPreheader
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:183
llvm::GlobalValue::setLinkage
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:530
ArrayRef.h
llvm::MDString::get
static MDString * get(LLVMContext &Context, StringRef Str)
Definition: Metadata.cpp:498
llvm::Function::Create
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:137
llvm::DiagnosticInfoPGOProfile
Diagnostic information for the PGO profiler.
Definition: DiagnosticInfo.h:274
llvm::LoopBase::hasDedicatedExits
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
Definition: LoopInfoImpl.h:112
llvm::getInstrProfRuntimeHookVarUseFuncName
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable.
Definition: InstrProf.h:160
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::InstrProfInstBase::getName
GlobalVariable * getName() const
Definition: IntrinsicInst.h:1322
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:82
llvm::ArrayType::get
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:638
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
llvm::collectPGOFuncNameStrings
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:438
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
Triple.h
llvm::getInstrProfRegFuncsName
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:130
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::LoopInfo
Definition: LoopInfo.h:1108
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1716
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
if
if(llvm_vc STREQUAL "") set(fake_version_inc "$
Definition: CMakeLists.txt:14
llvm::Instruction::getFunction
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:73
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:532
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:97
uint32_t
llvm::ConstantDataArray::getString
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2967
S
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
Definition: README.txt:210
llvm::DebugInfoCorrelate
cl::opt< bool > DebugInfoCorrelate("debug-info-correlate", cl::desc("Use debug info to correlate profiles."), cl::init(false))
Definition: PGOInstrumentation.cpp:325
SSAUpdater.h
BlockFrequencyInfo.h
llvm::AMDGPUISD::BFI
@ BFI
Definition: AMDGPUISelLowering.h:432
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:173
llvm::Init
Definition: Record.h:281
llvm::AMDGPU::SendMsg::Msg
const CustomOperand< const MCSubtargetInfo & > Msg[]
Definition: AMDGPUAsmUtils.cpp:39
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:142
llvm::BasicBlock::front
const Instruction & front() const
Definition: BasicBlock.h:318
llvm::DoInstrProfNameCompression
cl::opt< bool > DoInstrProfNameCompression
llvm::LoopInfoBase::getLoopsInPreorder
SmallVector< LoopT *, 4 > getLoopsInPreorder() const
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
Definition: LoopInfoImpl.h:587
llvm::StringRef::size
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
Attributes.h
Constant.h
llvm::LLVMContext::diagnose
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Definition: LLVMContext.cpp:248
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:240
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::Constant::getNullValue
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
Definition: Constants.cpp:350
get
Should compile to something r4 addze r3 instead we get
Definition: README.txt:24
NC
#define NC
Definition: regutils.h:42
llvm::toString
const char * toString(DWARFSectionKind Kind)
Definition: DWARFUnitIndex.h:67
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
llvm::Constant::isZeroValue
bool isZeroValue() const
Return true if the value is negative zero or null value.
Definition: Constants.cpp:62
llvm::Error
Lightweight error class with error context and mandatory checking.
Definition: Error.h:155
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
GlobalVariable.h
getOrInsertValueProfilingCall
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, ValueProfilingCallType CallType=ValueProfilingCallType::Default)
Definition: InstrProfiling.cpp:584
Casting.h
DiagnosticInfo.h
Function.h
llvm::Type::getPointerTo
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:774
enablesValueProfiling
static bool enablesValueProfiling(const Module &M)
Definition: InstrProfiling.cpp:779
llvm::TargetLibraryInfo
Provides information about what library functions are available for the current target.
Definition: TargetLibraryInfo.h:225
llvm::TargetStackID::Default
@ Default
Definition: TargetFrameLowering.h:28
llvm::ConstantArray::get
static Constant * get(ArrayType *T, ArrayRef< Constant * > V)
Definition: Constants.cpp:1241
llvm::InstrProfiling::run
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Definition: InstrProfiling.cpp:401
llvm::InstrProfCoverInst
This represents the llvm.instrprof.cover intrinsic.
Definition: IntrinsicInst.h:1337
llvm::MCID::Add
@ Add
Definition: MCInstrDesc.h:185
llvm::makeArrayRef
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:475
llvm::InstrProfIncrementInst
This represents the llvm.instrprof.increment intrinsic.
Definition: IntrinsicInst.h:1348
llvm::getInstrProfSectionName
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:212
llvm::GlobalValue::ExternalLinkage
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
llvm::Type::getVoidTy
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:222
llvm::GlobalValue::PrivateLinkage
@ PrivateLinkage
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Instructions.h
llvm::getInstrProfValueProfMemOpFuncName
StringRef getInstrProfValueProfMemOpFuncName()
Return the name profile runtime entry point to do memop size value profiling.
Definition: InstrProf.h:85
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
needsRuntimeRegistrationOfSectionRange
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
Definition: InstrProfiling.cpp:826
SmallVector.h
llvm::InstrProfOptions::NoRedZone
bool NoRedZone
Definition: Instrumentation.h:108
Dominators.h
llvm::getCoverageUnusedNamesVarName
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:121
ModuleUtils.h
llvm::getInstrProfCounterBiasVarName
StringRef getInstrProfCounterBiasVarName()
Definition: InstrProf.h:164
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::FunctionCallee
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:119
llvm::SmallVectorImpl< BasicBlock * >
InstrProfCorrelator.h
llvm::reverse
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:485
llvm::Type::getInt16Ty
static IntegerType * getInt16Ty(LLVMContext &C)
Definition: Type.cpp:238
DerivedTypes.h
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
llvm::InnerAnalysisManagerProxy
An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...
Definition: PassManager.h:931
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1473
llvm::getInstrProfValueProfFuncName
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:79
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:394
llvm::appendToGlobalCtors
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:65
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::getInstrProfDataVarPrefix
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:93
llvm::cl::desc
Definition: CommandLine.h:413
llvm::GlobalObject::setAlignment
void setAlignment(MaybeAlign Align)
Definition: Globals.cpp:126
llvm::SSAUpdater
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition: SSAUpdater.h:38
llvm::GlobalValue::LinkOnceODRLinkage
@ LinkOnceODRLinkage
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
InitializePasses.h
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::getInstrProfRegFuncName
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:136
llvm::TargetLibraryAnalysis
Analysis pass providing the TargetLibraryInfo.
Definition: TargetLibraryInfo.h:449
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365