LLVM  9.0.0svn
InstrProfiling.cpp
Go to the documentation of this file.
1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10 // It also builds the data structures and initialization code needed for
11 // updating execution counts and emitting the profile at runtime.
12 //
13 //===----------------------------------------------------------------------===//
14 
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/ADT/Twine.h"
21 #include "llvm/Analysis/LoopInfo.h"
23 #include "llvm/IR/Attributes.h"
24 #include "llvm/IR/BasicBlock.h"
25 #include "llvm/IR/Constant.h"
26 #include "llvm/IR/Constants.h"
27 #include "llvm/IR/DerivedTypes.h"
28 #include "llvm/IR/Dominators.h"
29 #include "llvm/IR/Function.h"
30 #include "llvm/IR/GlobalValue.h"
31 #include "llvm/IR/GlobalVariable.h"
32 #include "llvm/IR/IRBuilder.h"
33 #include "llvm/IR/Instruction.h"
34 #include "llvm/IR/Instructions.h"
35 #include "llvm/IR/IntrinsicInst.h"
36 #include "llvm/IR/Module.h"
37 #include "llvm/IR/Type.h"
38 #include "llvm/Pass.h"
40 #include "llvm/Support/Casting.h"
42 #include "llvm/Support/Error.h"
47 #include <algorithm>
48 #include <cassert>
49 #include <cstddef>
50 #include <cstdint>
51 #include <string>
52 
53 using namespace llvm;
54 
55 #define DEBUG_TYPE "instrprof"
56 
57 // The start and end values of precise value profile range for memory
58 // intrinsic sizes
60  "memop-size-range",
61  cl::desc("Set the range of size in memory intrinsic calls to be profiled "
62  "precisely, in a format of <start_val>:<end_val>"),
63  cl::init(""));
64 
65 // The value that considered to be large value in memory intrinsic.
67  "memop-size-large",
68  cl::desc("Set large value thresthold in memory intrinsic size profiling. "
69  "Value of 0 disables the large value profiling."),
70  cl::init(8192));
71 
72 namespace {
73 
74 cl::opt<bool> DoNameCompression("enable-name-compression",
75  cl::desc("Enable name string compression"),
76  cl::init(true));
77 
78 cl::opt<bool> DoHashBasedCounterSplit(
79  "hash-based-counter-split",
80  cl::desc("Rename counter variable of a comdat function based on cfg hash"),
81  cl::init(true));
82 
83 cl::opt<bool> ValueProfileStaticAlloc(
84  "vp-static-alloc",
85  cl::desc("Do static counter allocation for value profiler"),
86  cl::init(true));
87 
88 cl::opt<double> NumCountersPerValueSite(
89  "vp-counters-per-site",
90  cl::desc("The average number of profile counters allocated "
91  "per value profiling site."),
92  // This is set to a very small value because in real programs, only
93  // a very small percentage of value sites have non-zero targets, e.g, 1/30.
94  // For those sites with non-zero profile, the average number of targets
95  // is usually smaller than 2.
96  cl::init(1.0));
97 
98 cl::opt<bool> AtomicCounterUpdateAll(
99  "instrprof-atomic-counter-update-all", cl::ZeroOrMore,
100  cl::desc("Make all profile counter updates atomic (for testing only)"),
101  cl::init(false));
102 
103 cl::opt<bool> AtomicCounterUpdatePromoted(
104  "atomic-counter-update-promoted", cl::ZeroOrMore,
105  cl::desc("Do counter update using atomic fetch add "
106  " for promoted counters only"),
107  cl::init(false));
108 
109 // If the option is not specified, the default behavior about whether
110 // counter promotion is done depends on how instrumentaiton lowering
111 // pipeline is setup, i.e., the default value of true of this option
112 // does not mean the promotion will be done by default. Explicitly
113 // setting this option can override the default behavior.
114 cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore,
115  cl::desc("Do counter register promotion"),
116  cl::init(false));
117 cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
118  cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(20),
119  cl::desc("Max number counter promotions per loop to avoid"
120  " increasing register pressure too much"));
121 
122 // A debug option
124  MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1),
125  cl::desc("Max number of allowed counter promotions"));
126 
127 cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
128  cl::ZeroOrMore, "speculative-counter-promotion-max-exiting", cl::init(3),
129  cl::desc("The max number of exiting blocks of a loop to allow "
130  " speculative counter promotion"));
131 
132 cl::opt<bool> SpeculativeCounterPromotionToLoop(
133  cl::ZeroOrMore, "speculative-counter-promotion-to-loop", cl::init(false),
134  cl::desc("When the option is false, if the target block is in a loop, "
135  "the promotion will be disallowed unless the promoted counter "
136  " update can be further/iteratively promoted into an acyclic "
137  " region."));
138 
139 cl::opt<bool> IterativeCounterPromotion(
140  cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true),
141  cl::desc("Allow counter promotion across the whole loop nest."));
142 
143 class InstrProfilingLegacyPass : public ModulePass {
144  InstrProfiling InstrProf;
145 
146 public:
147  static char ID;
148 
149  InstrProfilingLegacyPass() : ModulePass(ID) {}
150  InstrProfilingLegacyPass(const InstrProfOptions &Options)
151  : ModulePass(ID), InstrProf(Options) {}
152 
153  StringRef getPassName() const override {
154  return "Frontend instrumentation-based coverage lowering";
155  }
156 
157  bool runOnModule(Module &M) override {
158  return InstrProf.run(M, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI());
159  }
160 
161  void getAnalysisUsage(AnalysisUsage &AU) const override {
162  AU.setPreservesCFG();
164  }
165 };
166 
167 ///
168 /// A helper class to promote one counter RMW operation in the loop
169 /// into register update.
170 ///
171 /// RWM update for the counter will be sinked out of the loop after
172 /// the transformation.
173 ///
174 class PGOCounterPromoterHelper : public LoadAndStorePromoter {
175 public:
176  PGOCounterPromoterHelper(
178  BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
179  ArrayRef<Instruction *> InsertPts,
181  LoopInfo &LI)
182  : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
183  InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
184  assert(isa<LoadInst>(L));
185  assert(isa<StoreInst>(S));
186  SSA.AddAvailableValue(PH, Init);
187  }
188 
189  void doExtraRewritesBeforeFinalDeletion() override {
190  for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
191  BasicBlock *ExitBlock = ExitBlocks[i];
192  Instruction *InsertPos = InsertPts[i];
193  // Get LiveIn value into the ExitBlock. If there are multiple
194  // predecessors, the value is defined by a PHI node in this
195  // block.
196  Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
197  Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
198  Type *Ty = LiveInValue->getType();
199  IRBuilder<> Builder(InsertPos);
200  if (AtomicCounterUpdatePromoted)
201  // automic update currently can only be promoted across the current
202  // loop, not the whole loop nest.
203  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
205  else {
206  LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
207  auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
208  auto *NewStore = Builder.CreateStore(NewVal, Addr);
209 
210  // Now update the parent loop's candidate list:
211  if (IterativeCounterPromotion) {
212  auto *TargetLoop = LI.getLoopFor(ExitBlock);
213  if (TargetLoop)
214  LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
215  }
216  }
217  }
218  }
219 
220 private:
222  ArrayRef<BasicBlock *> ExitBlocks;
223  ArrayRef<Instruction *> InsertPts;
225  LoopInfo &LI;
226 };
227 
228 /// A helper class to do register promotion for all profile counter
229 /// updates in a loop.
230 ///
231 class PGOCounterPromoter {
232 public:
233  PGOCounterPromoter(
235  Loop &CurLoop, LoopInfo &LI)
236  : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop),
237  LI(LI) {
238 
239  SmallVector<BasicBlock *, 8> LoopExitBlocks;
241  L.getExitBlocks(LoopExitBlocks);
242 
243  for (BasicBlock *ExitBlock : LoopExitBlocks) {
244  if (BlockSet.insert(ExitBlock).second) {
245  ExitBlocks.push_back(ExitBlock);
246  InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
247  }
248  }
249  }
250 
251  bool run(int64_t *NumPromoted) {
252  // Skip 'infinite' loops:
253  if (ExitBlocks.size() == 0)
254  return false;
255  unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
256  if (MaxProm == 0)
257  return false;
258 
259  unsigned Promoted = 0;
260  for (auto &Cand : LoopToCandidates[&L]) {
261 
263  SSAUpdater SSA(&NewPHIs);
264  Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
265 
266  PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
267  L.getLoopPreheader(), ExitBlocks,
268  InsertPts, LoopToCandidates, LI);
269  Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
270  Promoted++;
271  if (Promoted >= MaxProm)
272  break;
273 
274  (*NumPromoted)++;
275  if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
276  break;
277  }
278 
279  LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
280  << L.getLoopDepth() << ")\n");
281  return Promoted != 0;
282  }
283 
284 private:
285  bool allowSpeculativeCounterPromotion(Loop *LP) {
286  SmallVector<BasicBlock *, 8> ExitingBlocks;
287  L.getExitingBlocks(ExitingBlocks);
288  // Not considierered speculative.
289  if (ExitingBlocks.size() == 1)
290  return true;
291  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
292  return false;
293  return true;
294  }
295 
296  // Returns the max number of Counter Promotions for LP.
297  unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
298  // We can't insert into a catchswitch.
299  SmallVector<BasicBlock *, 8> LoopExitBlocks;
300  LP->getExitBlocks(LoopExitBlocks);
301  if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
302  return isa<CatchSwitchInst>(Exit->getTerminator());
303  }))
304  return 0;
305 
306  if (!LP->hasDedicatedExits())
307  return 0;
308 
309  BasicBlock *PH = LP->getLoopPreheader();
310  if (!PH)
311  return 0;
312 
313  SmallVector<BasicBlock *, 8> ExitingBlocks;
314  LP->getExitingBlocks(ExitingBlocks);
315  // Not considierered speculative.
316  if (ExitingBlocks.size() == 1)
317  return MaxNumOfPromotionsPerLoop;
318 
319  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
320  return 0;
321 
322  // Whether the target block is in a loop does not matter:
323  if (SpeculativeCounterPromotionToLoop)
324  return MaxNumOfPromotionsPerLoop;
325 
326  // Now check the target block:
327  unsigned MaxProm = MaxNumOfPromotionsPerLoop;
328  for (auto *TargetBlock : LoopExitBlocks) {
329  auto *TargetLoop = LI.getLoopFor(TargetBlock);
330  if (!TargetLoop)
331  continue;
332  unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
333  unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
334  MaxProm =
335  std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
336  PendingCandsInTarget);
337  }
338  return MaxProm;
339  }
340 
342  SmallVector<BasicBlock *, 8> ExitBlocks;
344  Loop &L;
345  LoopInfo &LI;
346 };
347 
348 } // end anonymous namespace
349 
351  auto &TLI = AM.getResult<TargetLibraryAnalysis>(M);
352  if (!run(M, TLI))
353  return PreservedAnalyses::all();
354 
355  return PreservedAnalyses::none();
356 }
357 
360  InstrProfilingLegacyPass, "instrprof",
361  "Frontend instrumentation-based coverage lowering.", false, false)
364  InstrProfilingLegacyPass, "instrprof",
365  "Frontend instrumentation-based coverage lowering.", false, false)
366 
367 ModulePass *
369  return new InstrProfilingLegacyPass(Options);
370 }
371 
374  if (Inc)
375  return Inc;
376  return dyn_cast<InstrProfIncrementInst>(Instr);
377 }
378 
379 bool InstrProfiling::lowerIntrinsics(Function *F) {
380  bool MadeChange = false;
381  PromotionCandidates.clear();
382  for (BasicBlock &BB : *F) {
383  for (auto I = BB.begin(), E = BB.end(); I != E;) {
384  auto Instr = I++;
386  if (Inc) {
387  lowerIncrement(Inc);
388  MadeChange = true;
389  } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
390  lowerValueProfileInst(Ind);
391  MadeChange = true;
392  }
393  }
394  }
395 
396  if (!MadeChange)
397  return false;
398 
399  promoteCounterLoadStores(F);
400  return true;
401 }
402 
403 bool InstrProfiling::isCounterPromotionEnabled() const {
404  if (DoCounterPromotion.getNumOccurrences() > 0)
405  return DoCounterPromotion;
406 
407  return Options.DoCounterPromotion;
408 }
409 
410 void InstrProfiling::promoteCounterLoadStores(Function *F) {
411  if (!isCounterPromotionEnabled())
412  return;
413 
414  DominatorTree DT(*F);
415  LoopInfo LI(DT);
416  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
417 
418  for (const auto &LoadStore : PromotionCandidates) {
419  auto *CounterLoad = LoadStore.first;
420  auto *CounterStore = LoadStore.second;
421  BasicBlock *BB = CounterLoad->getParent();
422  Loop *ParentLoop = LI.getLoopFor(BB);
423  if (!ParentLoop)
424  continue;
425  LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
426  }
427 
429 
430  // Do a post-order traversal of the loops so that counter updates can be
431  // iteratively hoisted outside the loop nest.
432  for (auto *Loop : llvm::reverse(Loops)) {
433  PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI);
434  Promoter.run(&TotalCountersPromoted);
435  }
436 }
437 
438 /// Check if the module contains uses of any profiling intrinsics.
440  if (auto *F = M.getFunction(
441  Intrinsic::getName(llvm::Intrinsic::instrprof_increment)))
442  if (!F->use_empty())
443  return true;
444  if (auto *F = M.getFunction(
445  Intrinsic::getName(llvm::Intrinsic::instrprof_increment_step)))
446  if (!F->use_empty())
447  return true;
448  if (auto *F = M.getFunction(
449  Intrinsic::getName(llvm::Intrinsic::instrprof_value_profile)))
450  if (!F->use_empty())
451  return true;
452  return false;
453 }
454 
456  this->M = &M;
457  this->TLI = &TLI;
458  NamesVar = nullptr;
459  NamesSize = 0;
460  ProfileDataMap.clear();
461  UsedVars.clear();
462  getMemOPSizeRangeFromOption(MemOPSizeRange, MemOPSizeRangeStart,
463  MemOPSizeRangeLast);
464  TT = Triple(M.getTargetTriple());
465 
466  // Emit the runtime hook even if no counters are present.
467  bool MadeChange = emitRuntimeHook();
468 
469  // Improve compile time by avoiding linear scans when there is no work.
470  GlobalVariable *CoverageNamesVar =
472  if (!containsProfilingIntrinsics(M) && !CoverageNamesVar)
473  return MadeChange;
474 
475  // We did not know how many value sites there would be inside
476  // the instrumented function. This is counting the number of instrumented
477  // target value sites to enter it as field in the profile data variable.
478  for (Function &F : M) {
479  InstrProfIncrementInst *FirstProfIncInst = nullptr;
480  for (BasicBlock &BB : F)
481  for (auto I = BB.begin(), E = BB.end(); I != E; I++)
482  if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
483  computeNumValueSiteCounts(Ind);
484  else if (FirstProfIncInst == nullptr)
485  FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
486 
487  // Value profiling intrinsic lowering requires per-function profile data
488  // variable to be created first.
489  if (FirstProfIncInst != nullptr)
490  static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
491  }
492 
493  for (Function &F : M)
494  MadeChange |= lowerIntrinsics(&F);
495 
496  if (CoverageNamesVar) {
497  lowerCoverageData(CoverageNamesVar);
498  MadeChange = true;
499  }
500 
501  if (!MadeChange)
502  return false;
503 
504  emitVNodes();
505  emitNameData();
506  emitRegistration();
507  emitUses();
508  emitInitialization();
509  return true;
510 }
511 
512 static FunctionCallee
514  bool IsRange = false) {
515  LLVMContext &Ctx = M.getContext();
516  auto *ReturnTy = Type::getVoidTy(M.getContext());
517 
519  if (auto AK = TLI.getExtAttrForI32Param(false))
520  AL = AL.addParamAttribute(M.getContext(), 2, AK);
521 
522  if (!IsRange) {
523  Type *ParamTypes[] = {
524 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
526  };
527  auto *ValueProfilingCallTy =
528  FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
530  ValueProfilingCallTy, AL);
531  } else {
532  Type *RangeParamTypes[] = {
533 #define VALUE_RANGE_PROF 1
534 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
536 #undef VALUE_RANGE_PROF
537  };
538  auto *ValueRangeProfilingCallTy =
539  FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
541  ValueRangeProfilingCallTy, AL);
542  }
543 }
544 
545 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
546  GlobalVariable *Name = Ind->getName();
547  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
548  uint64_t Index = Ind->getIndex()->getZExtValue();
549  auto It = ProfileDataMap.find(Name);
550  if (It == ProfileDataMap.end()) {
551  PerFunctionProfileData PD;
552  PD.NumValueSites[ValueKind] = Index + 1;
553  ProfileDataMap[Name] = PD;
554  } else if (It->second.NumValueSites[ValueKind] <= Index)
555  It->second.NumValueSites[ValueKind] = Index + 1;
556 }
557 
558 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
559  GlobalVariable *Name = Ind->getName();
560  auto It = ProfileDataMap.find(Name);
561  assert(It != ProfileDataMap.end() && It->second.DataVar &&
562  "value profiling detected in function with no counter incerement");
563 
564  GlobalVariable *DataVar = It->second.DataVar;
565  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
566  uint64_t Index = Ind->getIndex()->getZExtValue();
567  for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
568  Index += It->second.NumValueSites[Kind];
569 
570  IRBuilder<> Builder(Ind);
571  bool IsRange = (Ind->getValueKind()->getZExtValue() ==
572  llvm::InstrProfValueKind::IPVK_MemOPSize);
573  CallInst *Call = nullptr;
574  if (!IsRange) {
575  Value *Args[3] = {Ind->getTargetValue(),
576  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
577  Builder.getInt32(Index)};
578  Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args);
579  } else {
580  Value *Args[6] = {
581  Ind->getTargetValue(),
582  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
583  Builder.getInt32(Index),
584  Builder.getInt64(MemOPSizeRangeStart),
585  Builder.getInt64(MemOPSizeRangeLast),
586  Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
587  Call =
588  Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true), Args);
589  }
590  if (auto AK = TLI->getExtAttrForI32Param(false))
591  Call->addParamAttr(2, AK);
592  Ind->replaceAllUsesWith(Call);
593  Ind->eraseFromParent();
594 }
595 
596 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
597  GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
598 
599  IRBuilder<> Builder(Inc);
600  uint64_t Index = Inc->getIndex()->getZExtValue();
601  Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(),
602  Counters, 0, Index);
603 
604  if (Options.Atomic || AtomicCounterUpdateAll) {
605  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
607  } else {
608  Value *IncStep = Inc->getStep();
609  Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
610  auto *Count = Builder.CreateAdd(Load, Inc->getStep());
611  auto *Store = Builder.CreateStore(Count, Addr);
612  if (isCounterPromotionEnabled())
613  PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
614  }
615  Inc->eraseFromParent();
616 }
617 
618 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
619  ConstantArray *Names =
620  cast<ConstantArray>(CoverageNamesVar->getInitializer());
621  for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
622  Constant *NC = Names->getOperand(I);
623  Value *V = NC->stripPointerCasts();
624  assert(isa<GlobalVariable>(V) && "Missing reference to function name");
625  GlobalVariable *Name = cast<GlobalVariable>(V);
626 
628  ReferencedNames.push_back(Name);
629  NC->dropAllReferences();
630  }
631  CoverageNamesVar->eraseFromParent();
632 }
633 
634 /// Get the name of a profiling variable for a particular function.
636  StringRef NamePrefix = getInstrProfNameVarPrefix();
637  StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
638  Function *F = Inc->getParent()->getParent();
639  Module *M = F->getParent();
640  if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
641  !canRenameComdatFunc(*F))
642  return (Prefix + Name).str();
643  uint64_t FuncHash = Inc->getHash()->getZExtValue();
644  SmallVector<char, 24> HashPostfix;
645  if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
646  return (Prefix + Name).str();
647  return (Prefix + Name + "." + Twine(FuncHash)).str();
648 }
649 
650 static inline bool shouldRecordFunctionAddr(Function *F) {
651  // Check the linkage
652  bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
653  if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
654  !HasAvailableExternallyLinkage)
655  return true;
656 
657  // A function marked 'alwaysinline' with available_externally linkage can't
658  // have its address taken. Doing so would create an undefined external ref to
659  // the function, which would fail to link.
660  if (HasAvailableExternallyLinkage &&
661  F->hasFnAttribute(Attribute::AlwaysInline))
662  return false;
663 
664  // Prohibit function address recording if the function is both internal and
665  // COMDAT. This avoids the profile data variable referencing internal symbols
666  // in COMDAT.
667  if (F->hasLocalLinkage() && F->hasComdat())
668  return false;
669 
670  // Check uses of this function for other than direct calls or invokes to it.
671  // Inline virtual functions have linkeOnceODR linkage. When a key method
672  // exists, the vtable will only be emitted in the TU where the key method
673  // is defined. In a TU where vtable is not available, the function won't
674  // be 'addresstaken'. If its address is not recorded here, the profile data
675  // with missing address may be picked by the linker leading to missing
676  // indirect call target info.
677  return F->hasAddressTaken() || F->hasLinkOnceLinkage();
678 }
679 
681  // Don't do this for Darwin. compiler-rt uses linker magic.
682  if (TT.isOSDarwin())
683  return false;
684 
685  // Use linker script magic to get data/cnts/name start/end.
686  if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
687  TT.isOSFuchsia() || TT.isPS4CPU() || TT.isOSWindows())
688  return false;
689 
690  return true;
691 }
692 
694 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
695  GlobalVariable *NamePtr = Inc->getName();
696  auto It = ProfileDataMap.find(NamePtr);
697  PerFunctionProfileData PD;
698  if (It != ProfileDataMap.end()) {
699  if (It->second.RegionCounters)
700  return It->second.RegionCounters;
701  PD = It->second;
702  }
703 
704  // Match the linkage and visibility of the name global, except on COFF, where
705  // the linkage must be local and consequentially the visibility must be
706  // default.
707  Function *Fn = Inc->getParent()->getParent();
708  GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
710  if (TT.isOSBinFormatCOFF()) {
712  Visibility = GlobalValue::DefaultVisibility;
713  }
714 
715  // Move the name variable to the right section. Place them in a COMDAT group
716  // if the associated function is a COMDAT. This will make sure that only one
717  // copy of counters of the COMDAT function will be emitted after linking.
718  Comdat *Cmdt = nullptr;
719  GlobalValue::LinkageTypes CounterLinkage = Linkage;
720  if (needsComdatForCounter(*Fn, *M)) {
721  if (TT.isOSBinFormatCOFF()) {
722  // There are two cases that need a comdat on COFF:
723  // 1. Functions that already have comdats (standard case)
724  // 2. available_externally functions (dllimport and C99 inline)
725  // In the first case, put all the data in the original function comdat. In
726  // the second case, create a new comdat group using the counter as the
727  // leader. It's linkage must be external, so use linkonce_odr linkage in
728  // that case.
729  if (Comdat *C = Fn->getComdat()) {
730  Cmdt = C;
731  } else {
732  Cmdt = M->getOrInsertComdat(
734  CounterLinkage = GlobalValue::LinkOnceODRLinkage;
735  }
736  } else {
737  // For other platforms that use comdats (ELF), make a new comdat group for
738  // all the profile data. It will be deduplicated within the current DSO.
740  }
741  }
742 
743  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
744  LLVMContext &Ctx = M->getContext();
745  ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
746 
747  // Create the counters variable.
748  auto *CounterPtr =
749  new GlobalVariable(*M, CounterTy, false, Linkage,
750  Constant::getNullValue(CounterTy),
752  CounterPtr->setVisibility(Visibility);
753  CounterPtr->setSection(
754  getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
755  CounterPtr->setAlignment(8);
756  CounterPtr->setComdat(Cmdt);
757  CounterPtr->setLinkage(CounterLinkage);
758 
759  auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
760  // Allocate statically the array of pointers to value profile nodes for
761  // the current function.
762  Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
763  if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(TT)) {
764  uint64_t NS = 0;
765  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
766  NS += PD.NumValueSites[Kind];
767  if (NS) {
768  ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
769 
770  auto *ValuesVar =
771  new GlobalVariable(*M, ValuesTy, false, Linkage,
772  Constant::getNullValue(ValuesTy),
774  ValuesVar->setVisibility(Visibility);
775  ValuesVar->setSection(
776  getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
777  ValuesVar->setAlignment(8);
778  ValuesVar->setComdat(Cmdt);
779  ValuesPtrExpr =
781  }
782  }
783 
784  // Create data variable.
785  auto *Int16Ty = Type::getInt16Ty(Ctx);
786  auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
787  Type *DataTypes[] = {
788 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
790  };
791  auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
792 
793  Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
794  ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
795  : ConstantPointerNull::get(Int8PtrTy);
796 
797  Constant *Int16ArrayVals[IPVK_Last + 1];
798  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
799  Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
800 
801  Constant *DataVals[] = {
802 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
804  };
805  auto *Data = new GlobalVariable(*M, DataTy, false, Linkage,
806  ConstantStruct::get(DataTy, DataVals),
808  Data->setVisibility(Visibility);
809  Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
810  Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
811  Data->setComdat(Cmdt);
812 
813  PD.RegionCounters = CounterPtr;
814  PD.DataVar = Data;
815  ProfileDataMap[NamePtr] = PD;
816 
817  // Mark the data variable as used so that it isn't stripped out.
818  UsedVars.push_back(Data);
819  // Now that the linkage set by the FE has been passed to the data and counter
820  // variables, reset Name variable's linkage and visibility to private so that
821  // it can be removed later by the compiler.
823  // Collect the referenced names to be used by emitNameData.
824  ReferencedNames.push_back(NamePtr);
825 
826  return CounterPtr;
827 }
828 
829 void InstrProfiling::emitVNodes() {
830  if (!ValueProfileStaticAlloc)
831  return;
832 
833  // For now only support this on platforms that do
834  // not require runtime registration to discover
835  // named section start/end.
837  return;
838 
839  size_t TotalNS = 0;
840  for (auto &PD : ProfileDataMap) {
841  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
842  TotalNS += PD.second.NumValueSites[Kind];
843  }
844 
845  if (!TotalNS)
846  return;
847 
848  uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
849 // Heuristic for small programs with very few total value sites.
850 // The default value of vp-counters-per-site is chosen based on
851 // the observation that large apps usually have a low percentage
852 // of value sites that actually have any profile data, and thus
853 // the average number of counters per site is low. For small
854 // apps with very few sites, this may not be true. Bump up the
855 // number of counters in this case.
856 #define INSTR_PROF_MIN_VAL_COUNTS 10
857  if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
858  NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
859 
860  auto &Ctx = M->getContext();
861  Type *VNodeTypes[] = {
862 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
864  };
865  auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
866 
867  ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
868  auto *VNodesVar = new GlobalVariable(
869  *M, VNodesTy, false, GlobalValue::PrivateLinkage,
871  VNodesVar->setSection(
872  getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
873  UsedVars.push_back(VNodesVar);
874 }
875 
876 void InstrProfiling::emitNameData() {
877  std::string UncompressedData;
878 
879  if (ReferencedNames.empty())
880  return;
881 
882  std::string CompressedNameStr;
883  if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
884  DoNameCompression)) {
885  report_fatal_error(toString(std::move(E)), false);
886  }
887 
888  auto &Ctx = M->getContext();
889  auto *NamesVal = ConstantDataArray::getString(
890  Ctx, StringRef(CompressedNameStr), false);
891  NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
892  GlobalValue::PrivateLinkage, NamesVal,
894  NamesSize = CompressedNameStr.size();
895  NamesVar->setSection(
896  getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
897  // On COFF, it's important to reduce the alignment down to 1 to prevent the
898  // linker from inserting padding before the start of the names section or
899  // between names entries.
900  NamesVar->setAlignment(1);
901  UsedVars.push_back(NamesVar);
902 
903  for (auto *NamePtr : ReferencedNames)
904  NamePtr->eraseFromParent();
905 }
906 
907 void InstrProfiling::emitRegistration() {
909  return;
910 
911  // Construct the function.
912  auto *VoidTy = Type::getVoidTy(M->getContext());
913  auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
914  auto *Int64Ty = Type::getInt64Ty(M->getContext());
915  auto *RegisterFTy = FunctionType::get(VoidTy, false);
916  auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
918  RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
919  if (Options.NoRedZone)
920  RegisterF->addFnAttr(Attribute::NoRedZone);
921 
922  auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
923  auto *RuntimeRegisterF =
926 
927  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
928  for (Value *Data : UsedVars)
929  if (Data != NamesVar && !isa<Function>(Data))
930  IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
931 
932  if (NamesVar) {
933  Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
934  auto *NamesRegisterTy =
935  FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
936  auto *NamesRegisterF =
939  IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
940  IRB.getInt64(NamesSize)});
941  }
942 
943  IRB.CreateRetVoid();
944 }
945 
946 bool InstrProfiling::emitRuntimeHook() {
947  // We expect the linker to be invoked with -u<hook_var> flag for linux,
948  // for which case there is no need to emit the user function.
949  if (TT.isOSLinux())
950  return false;
951 
952  // If the module's provided its own runtime, we don't need to do anything.
954  return false;
955 
956  // Declare an external variable that will pull in the runtime initialization.
957  auto *Int32Ty = Type::getInt32Ty(M->getContext());
958  auto *Var =
960  nullptr, getInstrProfRuntimeHookVarName());
961 
962  // Make a function that uses it.
966  User->addFnAttr(Attribute::NoInline);
967  if (Options.NoRedZone)
968  User->addFnAttr(Attribute::NoRedZone);
969  User->setVisibility(GlobalValue::HiddenVisibility);
970  if (TT.supportsCOMDAT())
971  User->setComdat(M->getOrInsertComdat(User->getName()));
972 
974  auto *Load = IRB.CreateLoad(Int32Ty, Var);
975  IRB.CreateRet(Load);
976 
977  // Mark the user variable as used so that it isn't stripped out.
978  UsedVars.push_back(User);
979  return true;
980 }
981 
982 void InstrProfiling::emitUses() {
983  if (!UsedVars.empty())
984  appendToUsed(*M, UsedVars);
985 }
986 
987 void InstrProfiling::emitInitialization() {
988  // Create variable for profile name.
989  createProfileFileNameVar(*M, Options.InstrProfileOutput);
990  Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
991  if (!RegisterF)
992  return;
993 
994  // Create the initialization function.
995  auto *VoidTy = Type::getVoidTy(M->getContext());
996  auto *F = Function::Create(FunctionType::get(VoidTy, false),
999  F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1000  F->addFnAttr(Attribute::NoInline);
1001  if (Options.NoRedZone)
1002  F->addFnAttr(Attribute::NoRedZone);
1003 
1004  // Add the basic block and the necessary calls.
1005  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1006  IRB.CreateCall(RegisterF, {});
1007  IRB.CreateRetVoid();
1008 
1009  appendToGlobalCtors(*M, F, 0);
1010 }
cl::opt< std::string > MemOPSizeRange("memop-size-range", cl::desc("Set the range of size in memory intrinsic calls to be profiled " "precisely, in a format of <start_val>:<end_val>"), cl::init(""))
uint64_t CallInst * C
bool isOSDarwin() const
isOSDarwin - Is this a "Darwin" OS (OS X, iOS, or watchOS).
Definition: Triple.h:474
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:67
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable...
Definition: InstrProf.h:156
const std::string & getTargetTriple() const
Get the target triple which is a string describing the target host.
Definition: Module.h:239
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2566
bool hasLocalLinkage() const
Definition: GlobalValue.h:435
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition: SSAUpdater.h:38
Value * getPointerOperand(Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
ConstantInt * getIndex() const
void dropAllReferences()
Drop all references to operands.
Definition: User.h:294
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:82
LLVM_NODISCARD bool endswith(StringRef Suffix) const
Check if this string ends with the given Suffix.
Definition: StringRef.h:267
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:769
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
cl::opt< unsigned > MemOPSizeLarge("memop-size-large", cl::desc("Set large value thresthold in memory intrinsic size profiling. " "Value of 0 disables the large value profiling."), cl::init(8192))
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:126
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve &#39;CreateLoad(Ty, Ptr, "...")&#39; correctly, instead of converting the string to &#39;bool...
Definition: IRBuilder.h:1392
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:64
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
Definition: LoopInfoImpl.h:85
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:164
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
#define INSTR_PROF_MIN_VAL_COUNTS
This class represents a function call, abstracting a target machine&#39;s calling convention.
bool hasAvailableExternallyLinkage() const
Definition: GlobalValue.h:422
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:971
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:173
const GlobalVariable * getNamedGlobal(StringRef Name) const
Return the global variable in the module with the specified name, of arbitrary type.
Definition: Module.h:404
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Externally visible function.
Definition: GlobalValue.h:48
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:320
GlobalVariable * getGlobalVariable(StringRef Name) const
Look up the specified global variable in the module symbol table.
Definition: Module.h:389
This file provides the interface for LLVM&#39;s PGO Instrumentation lowering pass.
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:164
F(f)
bool isOSFuchsia() const
Definition: Triple.h:494
An instruction for reading from memory.
Definition: Instructions.h:167
ELFYAML::ELF_STV Visibility
Definition: ELFYAML.cpp:807
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:176
ModulePass * createInstrProfilingLegacyPass(const InstrProfOptions &Options=InstrProfOptions())
Insert frontend instrumentation based profiling.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:137
static IntegerType * getInt16Ty(LLVMContext &C)
Definition: Type.cpp:174
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:91
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:264
Attribute::AttrKind getExtAttrForI32Param(bool Signed=true) const
Returns extension attribute kind to be used for i32 parameters corresponding to C-level int or unsign...
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
Hexagon Hardware Loops
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:145
std::string toString(Error E)
Write all error messages (if any) in E to a string.
Definition: Error.h:966
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:625
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:689
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:85
#define INT64_MIN
Definition: DataTypes.h:80
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:243
Instrumentation based profiling lowering pass.
ReturnInst * CreateRet(Value *V)
Create a &#39;ret <val>&#39; instruction.
Definition: IRBuilder.h:828
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:742
This file contains the simple types necessary to represent the attributes associated with functions a...
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:378
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1049
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:117
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:578
static StructType * get(LLVMContext &Context, ArrayRef< Type *> Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:341
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:137
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1421
void getExitBlocks(SmallVectorImpl< BlockT *> &ExitBlocks) const
Return all of the successor blocks of this loop.
Definition: LoopInfoImpl.h:62
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:266
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1767
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
void appendToUsed(Module &M, ArrayRef< GlobalValue *> Values)
Adds global values to the llvm.used list.
Class to represent array types.
Definition: DerivedTypes.h:400
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:62
Memory SSA
Definition: MemorySSA.cpp:64
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:130
LinkageTypes getLinkage() const
Definition: GlobalValue.h:450
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:429
bool isOSNetBSD() const
Definition: Triple.h:482
bool hasLinkOnceLinkage() const
Definition: GlobalValue.h:425
GlobalVariable * getName() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:144
Value * getOperand(unsigned i) const
Definition: User.h:169
ConstantInt * getNumCounters() const
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:156
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:951
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, bool IsRange=false)
static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix)
Get the name of a profiling variable for a particular function.
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:1772
bool isOSWindows() const
Tests whether the OS is Windows.
Definition: Triple.h:527
void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, int64_t &RangeLast)
Definition: InstrProf.cpp:994
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:925
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:422
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:135
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:148
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:153
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:216
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1400
VisibilityTypes getVisibility() const
Definition: GlobalValue.h:232
ConstantInt * getIndex() const
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1015
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:72
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
ValueKind
Value kinds.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:370
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function&#39;s PGO names...
Definition: InstrProf.h:102
void eraseFromParent()
eraseFromParent - This method unlinks &#39;this&#39; from the containing module and deletes it...
Definition: Globals.cpp:358
Represent the analysis usage information of a pass.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:160
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1192
bool isPS4CPU() const
Tests whether the target is the PS4 CPU.
Definition: Triple.h:622
static bool shouldRecordFunctionAddr(Function *F)
static FunctionType * get(Type *Result, ArrayRef< Type *> Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:296
static Constant * get(StructType *T, ArrayRef< Constant *> V)
Definition: Constants.cpp:1043
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:99
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:311
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Definition: IRBuilder.h:1487
void getExitingBlocks(SmallVectorImpl< BlockT *> &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
Definition: LoopInfoImpl.h:34
StringRef toStringRef(bool B)
Construct a string ref from a boolean.
Definition: StringExtras.h:52
const Constant * stripPointerCasts() const
Definition: Constant.h:173
Comdat * getOrInsertComdat(StringRef Name)
Return the Comdat in the module with the specified name.
Definition: Module.cpp:482
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:159
size_t size() const
Definition: SmallVector.h:52
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:219
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:43
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:88
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1311
unsigned getNumOperands() const
Definition: User.h:191
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:417
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
Helper class for promoting a collection of loads and stores into SSA Form using the SSAUpdater...
Definition: SSAUpdater.h:136
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:839
Module.h This file contains the declarations for the Module class.
Provides information about what library functions are available for the current target.
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:575
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:621
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:83
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:285
FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T, AttributeList AttributeList)
Look up the specified function in the module symbol table.
Definition: Module.cpp:143
#define NC
Definition: regutils.h:42
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:444
bool isOSFreeBSD() const
Definition: Triple.h:490
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition: Module.cpp:174
ConstantArray - Constant Array Declarations.
Definition: Constants.h:413
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:150
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:132
Options for the frontend instrumentation based profiling pass.
bool hasComdat() const
Definition: GlobalObject.h:99
This represents the llvm.instrprof_increment intrinsic.
const Comdat * getComdat() const
Definition: GlobalObject.h:100
ReturnInst * CreateRetVoid()
Create a &#39;ret void&#39; instruction.
Definition: IRBuilder.h:823
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:175
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:464
ConstantInt * getHash() const
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:106
#define I(x, y, z)
Definition: MD5.cpp:58
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:224
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:580
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:322
Type * getValueType() const
Definition: GlobalValue.h:275
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2009
SmallVector< LoopT *, 4 > getLoopsInPreorder()
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
Definition: LoopInfoImpl.h:582
Analysis pass providing the TargetLibraryInfo.
const unsigned Kind
bool hasAddressTaken(const User **=nullptr) const
hasAddressTaken - returns true if there are any uses of this function other than direct calls or invo...
Definition: Function.cpp:1257
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
StringRef getInstrProfValueRangeProfFuncName()
Return the name profile runtime entry point to do value range profiling.
Definition: InstrProf.h:77
aarch64 promote const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:565
LLVM Value Representation.
Definition: Value.h:72
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:94
GlobalVariable * getName() const
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
ConstantInt * getValueKind() const
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
A container for analyses that lazily runs them and caches their results.
StringRef getInstrProfComdatPrefix()
Return the name prefix of the COMDAT group for instrumentation variables associated with a COMDAT fun...
Definition: InstrProf.h:98
static bool lowerIntrinsics(Module &M)
#define LLVM_DEBUG(X)
Definition: Debug.h:122
This represents the llvm.instrprof_value_profile intrinsic.
static InstrProfIncrementInst * castToIncrementInst(Instruction *Instr)
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:188
bool use_empty() const
Definition: Value.h:322
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
IntegerType * Int32Ty
Value * CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1, const Twine &Name="")
Definition: IRBuilder.h:1648
INITIALIZE_PASS_BEGIN(InstrProfilingLegacyPass, "instrprof", "Frontend instrumentation-based coverage lowering.", false, false) INITIALIZE_PASS_END(InstrProfilingLegacyPass
const BasicBlock * getParent() const
Definition: Instruction.h:66