LLVM  9.0.0svn
InstrProfiling.cpp
Go to the documentation of this file.
1 //===-- InstrProfiling.cpp - Frontend instrumentation based profiling -----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass lowers instrprof_* intrinsics emitted by a frontend for profiling.
10 // It also builds the data structures and initialization code needed for
11 // updating execution counts and emitting the profile at runtime.
12 //
13 //===----------------------------------------------------------------------===//
14 
16 #include "llvm/ADT/ArrayRef.h"
17 #include "llvm/ADT/SmallVector.h"
18 #include "llvm/ADT/StringRef.h"
19 #include "llvm/ADT/Triple.h"
20 #include "llvm/ADT/Twine.h"
23 #include "llvm/Analysis/LoopInfo.h"
25 #include "llvm/IR/Attributes.h"
26 #include "llvm/IR/BasicBlock.h"
27 #include "llvm/IR/Constant.h"
28 #include "llvm/IR/Constants.h"
29 #include "llvm/IR/DerivedTypes.h"
30 #include "llvm/IR/Dominators.h"
31 #include "llvm/IR/Function.h"
32 #include "llvm/IR/GlobalValue.h"
33 #include "llvm/IR/GlobalVariable.h"
34 #include "llvm/IR/IRBuilder.h"
35 #include "llvm/IR/Instruction.h"
36 #include "llvm/IR/Instructions.h"
37 #include "llvm/IR/IntrinsicInst.h"
38 #include "llvm/IR/Module.h"
39 #include "llvm/IR/Type.h"
40 #include "llvm/Pass.h"
42 #include "llvm/Support/Casting.h"
44 #include "llvm/Support/Error.h"
49 #include <algorithm>
50 #include <cassert>
51 #include <cstddef>
52 #include <cstdint>
53 #include <string>
54 
55 using namespace llvm;
56 
57 #define DEBUG_TYPE "instrprof"
58 
59 // The start and end values of precise value profile range for memory
60 // intrinsic sizes
62  "memop-size-range",
63  cl::desc("Set the range of size in memory intrinsic calls to be profiled "
64  "precisely, in a format of <start_val>:<end_val>"),
65  cl::init(""));
66 
67 // The value that considered to be large value in memory intrinsic.
69  "memop-size-large",
70  cl::desc("Set large value thresthold in memory intrinsic size profiling. "
71  "Value of 0 disables the large value profiling."),
72  cl::init(8192));
73 
74 namespace {
75 
76 cl::opt<bool> DoNameCompression("enable-name-compression",
77  cl::desc("Enable name string compression"),
78  cl::init(true));
79 
80 cl::opt<bool> DoHashBasedCounterSplit(
81  "hash-based-counter-split",
82  cl::desc("Rename counter variable of a comdat function based on cfg hash"),
83  cl::init(true));
84 
85 cl::opt<bool> ValueProfileStaticAlloc(
86  "vp-static-alloc",
87  cl::desc("Do static counter allocation for value profiler"),
88  cl::init(true));
89 
90 cl::opt<double> NumCountersPerValueSite(
91  "vp-counters-per-site",
92  cl::desc("The average number of profile counters allocated "
93  "per value profiling site."),
94  // This is set to a very small value because in real programs, only
95  // a very small percentage of value sites have non-zero targets, e.g, 1/30.
96  // For those sites with non-zero profile, the average number of targets
97  // is usually smaller than 2.
98  cl::init(1.0));
99 
100 cl::opt<bool> AtomicCounterUpdateAll(
101  "instrprof-atomic-counter-update-all", cl::ZeroOrMore,
102  cl::desc("Make all profile counter updates atomic (for testing only)"),
103  cl::init(false));
104 
105 cl::opt<bool> AtomicCounterUpdatePromoted(
106  "atomic-counter-update-promoted", cl::ZeroOrMore,
107  cl::desc("Do counter update using atomic fetch add "
108  " for promoted counters only"),
109  cl::init(false));
110 
111 // If the option is not specified, the default behavior about whether
112 // counter promotion is done depends on how instrumentaiton lowering
113 // pipeline is setup, i.e., the default value of true of this option
114 // does not mean the promotion will be done by default. Explicitly
115 // setting this option can override the default behavior.
116 cl::opt<bool> DoCounterPromotion("do-counter-promotion", cl::ZeroOrMore,
117  cl::desc("Do counter register promotion"),
118  cl::init(false));
119 cl::opt<unsigned> MaxNumOfPromotionsPerLoop(
120  cl::ZeroOrMore, "max-counter-promotions-per-loop", cl::init(20),
121  cl::desc("Max number counter promotions per loop to avoid"
122  " increasing register pressure too much"));
123 
124 // A debug option
126  MaxNumOfPromotions(cl::ZeroOrMore, "max-counter-promotions", cl::init(-1),
127  cl::desc("Max number of allowed counter promotions"));
128 
129 cl::opt<unsigned> SpeculativeCounterPromotionMaxExiting(
130  cl::ZeroOrMore, "speculative-counter-promotion-max-exiting", cl::init(3),
131  cl::desc("The max number of exiting blocks of a loop to allow "
132  " speculative counter promotion"));
133 
134 cl::opt<bool> SpeculativeCounterPromotionToLoop(
135  cl::ZeroOrMore, "speculative-counter-promotion-to-loop", cl::init(false),
136  cl::desc("When the option is false, if the target block is in a loop, "
137  "the promotion will be disallowed unless the promoted counter "
138  " update can be further/iteratively promoted into an acyclic "
139  " region."));
140 
141 cl::opt<bool> IterativeCounterPromotion(
142  cl::ZeroOrMore, "iterative-counter-promotion", cl::init(true),
143  cl::desc("Allow counter promotion across the whole loop nest."));
144 
145 class InstrProfilingLegacyPass : public ModulePass {
146  InstrProfiling InstrProf;
147 
148 public:
149  static char ID;
150 
151  InstrProfilingLegacyPass() : ModulePass(ID) {}
152  InstrProfilingLegacyPass(const InstrProfOptions &Options, bool IsCS = false)
153  : ModulePass(ID), InstrProf(Options, IsCS) {}
154 
155  StringRef getPassName() const override {
156  return "Frontend instrumentation-based coverage lowering";
157  }
158 
159  bool runOnModule(Module &M) override {
160  return InstrProf.run(M, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI());
161  }
162 
163  void getAnalysisUsage(AnalysisUsage &AU) const override {
164  AU.setPreservesCFG();
166  }
167 };
168 
169 ///
170 /// A helper class to promote one counter RMW operation in the loop
171 /// into register update.
172 ///
173 /// RWM update for the counter will be sinked out of the loop after
174 /// the transformation.
175 ///
176 class PGOCounterPromoterHelper : public LoadAndStorePromoter {
177 public:
178  PGOCounterPromoterHelper(
180  BasicBlock *PH, ArrayRef<BasicBlock *> ExitBlocks,
181  ArrayRef<Instruction *> InsertPts,
183  LoopInfo &LI)
184  : LoadAndStorePromoter({L, S}, SSA), Store(S), ExitBlocks(ExitBlocks),
185  InsertPts(InsertPts), LoopToCandidates(LoopToCands), LI(LI) {
186  assert(isa<LoadInst>(L));
187  assert(isa<StoreInst>(S));
188  SSA.AddAvailableValue(PH, Init);
189  }
190 
191  void doExtraRewritesBeforeFinalDeletion() override {
192  for (unsigned i = 0, e = ExitBlocks.size(); i != e; ++i) {
193  BasicBlock *ExitBlock = ExitBlocks[i];
194  Instruction *InsertPos = InsertPts[i];
195  // Get LiveIn value into the ExitBlock. If there are multiple
196  // predecessors, the value is defined by a PHI node in this
197  // block.
198  Value *LiveInValue = SSA.GetValueInMiddleOfBlock(ExitBlock);
199  Value *Addr = cast<StoreInst>(Store)->getPointerOperand();
200  Type *Ty = LiveInValue->getType();
201  IRBuilder<> Builder(InsertPos);
202  if (AtomicCounterUpdatePromoted)
203  // automic update currently can only be promoted across the current
204  // loop, not the whole loop nest.
205  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, LiveInValue,
207  else {
208  LoadInst *OldVal = Builder.CreateLoad(Ty, Addr, "pgocount.promoted");
209  auto *NewVal = Builder.CreateAdd(OldVal, LiveInValue);
210  auto *NewStore = Builder.CreateStore(NewVal, Addr);
211 
212  // Now update the parent loop's candidate list:
213  if (IterativeCounterPromotion) {
214  auto *TargetLoop = LI.getLoopFor(ExitBlock);
215  if (TargetLoop)
216  LoopToCandidates[TargetLoop].emplace_back(OldVal, NewStore);
217  }
218  }
219  }
220  }
221 
222 private:
224  ArrayRef<BasicBlock *> ExitBlocks;
225  ArrayRef<Instruction *> InsertPts;
227  LoopInfo &LI;
228 };
229 
230 /// A helper class to do register promotion for all profile counter
231 /// updates in a loop.
232 ///
233 class PGOCounterPromoter {
234 public:
235  PGOCounterPromoter(
237  Loop &CurLoop, LoopInfo &LI, BlockFrequencyInfo *BFI)
238  : LoopToCandidates(LoopToCands), ExitBlocks(), InsertPts(), L(CurLoop),
239  LI(LI), BFI(BFI) {
240 
241  SmallVector<BasicBlock *, 8> LoopExitBlocks;
243  L.getExitBlocks(LoopExitBlocks);
244 
245  for (BasicBlock *ExitBlock : LoopExitBlocks) {
246  if (BlockSet.insert(ExitBlock).second) {
247  ExitBlocks.push_back(ExitBlock);
248  InsertPts.push_back(&*ExitBlock->getFirstInsertionPt());
249  }
250  }
251  }
252 
253  bool run(int64_t *NumPromoted) {
254  // Skip 'infinite' loops:
255  if (ExitBlocks.size() == 0)
256  return false;
257  unsigned MaxProm = getMaxNumOfPromotionsInLoop(&L);
258  if (MaxProm == 0)
259  return false;
260 
261  unsigned Promoted = 0;
262  for (auto &Cand : LoopToCandidates[&L]) {
263 
265  SSAUpdater SSA(&NewPHIs);
266  Value *InitVal = ConstantInt::get(Cand.first->getType(), 0);
267 
268  // If BFI is set, we will use it to guide the promotions.
269  if (BFI) {
270  auto *BB = Cand.first->getParent();
271  auto InstrCount = BFI->getBlockProfileCount(BB);
272  if (!InstrCount)
273  continue;
274  auto PreheaderCount = BFI->getBlockProfileCount(L.getLoopPreheader());
275  // If the average loop trip count is not greater than 1.5, we skip
276  // promotion.
277  if (PreheaderCount &&
278  (PreheaderCount.getValue() * 3) >= (InstrCount.getValue() * 2))
279  continue;
280  }
281 
282  PGOCounterPromoterHelper Promoter(Cand.first, Cand.second, SSA, InitVal,
283  L.getLoopPreheader(), ExitBlocks,
284  InsertPts, LoopToCandidates, LI);
285  Promoter.run(SmallVector<Instruction *, 2>({Cand.first, Cand.second}));
286  Promoted++;
287  if (Promoted >= MaxProm)
288  break;
289 
290  (*NumPromoted)++;
291  if (MaxNumOfPromotions != -1 && *NumPromoted >= MaxNumOfPromotions)
292  break;
293  }
294 
295  LLVM_DEBUG(dbgs() << Promoted << " counters promoted for loop (depth="
296  << L.getLoopDepth() << ")\n");
297  return Promoted != 0;
298  }
299 
300 private:
301  bool allowSpeculativeCounterPromotion(Loop *LP) {
302  SmallVector<BasicBlock *, 8> ExitingBlocks;
303  L.getExitingBlocks(ExitingBlocks);
304  // Not considierered speculative.
305  if (ExitingBlocks.size() == 1)
306  return true;
307  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
308  return false;
309  return true;
310  }
311 
312  // Returns the max number of Counter Promotions for LP.
313  unsigned getMaxNumOfPromotionsInLoop(Loop *LP) {
314  // We can't insert into a catchswitch.
315  SmallVector<BasicBlock *, 8> LoopExitBlocks;
316  LP->getExitBlocks(LoopExitBlocks);
317  if (llvm::any_of(LoopExitBlocks, [](BasicBlock *Exit) {
318  return isa<CatchSwitchInst>(Exit->getTerminator());
319  }))
320  return 0;
321 
322  if (!LP->hasDedicatedExits())
323  return 0;
324 
325  BasicBlock *PH = LP->getLoopPreheader();
326  if (!PH)
327  return 0;
328 
329  SmallVector<BasicBlock *, 8> ExitingBlocks;
330  LP->getExitingBlocks(ExitingBlocks);
331 
332  // If BFI is set, we do more aggressive promotions based on BFI.
333  if (BFI)
334  return (unsigned)-1;
335 
336  // Not considierered speculative.
337  if (ExitingBlocks.size() == 1)
338  return MaxNumOfPromotionsPerLoop;
339 
340  if (ExitingBlocks.size() > SpeculativeCounterPromotionMaxExiting)
341  return 0;
342 
343  // Whether the target block is in a loop does not matter:
344  if (SpeculativeCounterPromotionToLoop)
345  return MaxNumOfPromotionsPerLoop;
346 
347  // Now check the target block:
348  unsigned MaxProm = MaxNumOfPromotionsPerLoop;
349  for (auto *TargetBlock : LoopExitBlocks) {
350  auto *TargetLoop = LI.getLoopFor(TargetBlock);
351  if (!TargetLoop)
352  continue;
353  unsigned MaxPromForTarget = getMaxNumOfPromotionsInLoop(TargetLoop);
354  unsigned PendingCandsInTarget = LoopToCandidates[TargetLoop].size();
355  MaxProm =
356  std::min(MaxProm, std::max(MaxPromForTarget, PendingCandsInTarget) -
357  PendingCandsInTarget);
358  }
359  return MaxProm;
360  }
361 
363  SmallVector<BasicBlock *, 8> ExitBlocks;
365  Loop &L;
366  LoopInfo &LI;
368 };
369 
370 } // end anonymous namespace
371 
373  auto &TLI = AM.getResult<TargetLibraryAnalysis>(M);
374  if (!run(M, TLI))
375  return PreservedAnalyses::all();
376 
377  return PreservedAnalyses::none();
378 }
379 
382  InstrProfilingLegacyPass, "instrprof",
383  "Frontend instrumentation-based coverage lowering.", false, false)
386  InstrProfilingLegacyPass, "instrprof",
387  "Frontend instrumentation-based coverage lowering.", false, false)
388 
389 ModulePass *
391  bool IsCS) {
392  return new InstrProfilingLegacyPass(Options, IsCS);
393 }
394 
397  if (Inc)
398  return Inc;
399  return dyn_cast<InstrProfIncrementInst>(Instr);
400 }
401 
402 bool InstrProfiling::lowerIntrinsics(Function *F) {
403  bool MadeChange = false;
404  PromotionCandidates.clear();
405  for (BasicBlock &BB : *F) {
406  for (auto I = BB.begin(), E = BB.end(); I != E;) {
407  auto Instr = I++;
409  if (Inc) {
410  lowerIncrement(Inc);
411  MadeChange = true;
412  } else if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
413  lowerValueProfileInst(Ind);
414  MadeChange = true;
415  }
416  }
417  }
418 
419  if (!MadeChange)
420  return false;
421 
422  promoteCounterLoadStores(F);
423  return true;
424 }
425 
426 bool InstrProfiling::isCounterPromotionEnabled() const {
427  if (DoCounterPromotion.getNumOccurrences() > 0)
428  return DoCounterPromotion;
429 
430  return Options.DoCounterPromotion;
431 }
432 
433 void InstrProfiling::promoteCounterLoadStores(Function *F) {
434  if (!isCounterPromotionEnabled())
435  return;
436 
437  DominatorTree DT(*F);
438  LoopInfo LI(DT);
439  DenseMap<Loop *, SmallVector<LoadStorePair, 8>> LoopPromotionCandidates;
440 
441  std::unique_ptr<BlockFrequencyInfo> BFI;
442  if (Options.UseBFIInPromotion) {
443  std::unique_ptr<BranchProbabilityInfo> BPI;
444  BPI.reset(new BranchProbabilityInfo(*F, LI, TLI));
445  BFI.reset(new BlockFrequencyInfo(*F, *BPI, LI));
446  }
447 
448  for (const auto &LoadStore : PromotionCandidates) {
449  auto *CounterLoad = LoadStore.first;
450  auto *CounterStore = LoadStore.second;
451  BasicBlock *BB = CounterLoad->getParent();
452  Loop *ParentLoop = LI.getLoopFor(BB);
453  if (!ParentLoop)
454  continue;
455  LoopPromotionCandidates[ParentLoop].emplace_back(CounterLoad, CounterStore);
456  }
457 
459 
460  // Do a post-order traversal of the loops so that counter updates can be
461  // iteratively hoisted outside the loop nest.
462  for (auto *Loop : llvm::reverse(Loops)) {
463  PGOCounterPromoter Promoter(LoopPromotionCandidates, *Loop, LI, BFI.get());
464  Promoter.run(&TotalCountersPromoted);
465  }
466 }
467 
468 /// Check if the module contains uses of any profiling intrinsics.
470  if (auto *F = M.getFunction(
471  Intrinsic::getName(llvm::Intrinsic::instrprof_increment)))
472  if (!F->use_empty())
473  return true;
474  if (auto *F = M.getFunction(
475  Intrinsic::getName(llvm::Intrinsic::instrprof_increment_step)))
476  if (!F->use_empty())
477  return true;
478  if (auto *F = M.getFunction(
479  Intrinsic::getName(llvm::Intrinsic::instrprof_value_profile)))
480  if (!F->use_empty())
481  return true;
482  return false;
483 }
484 
486  this->M = &M;
487  this->TLI = &TLI;
488  NamesVar = nullptr;
489  NamesSize = 0;
490  ProfileDataMap.clear();
491  UsedVars.clear();
492  getMemOPSizeRangeFromOption(MemOPSizeRange, MemOPSizeRangeStart,
493  MemOPSizeRangeLast);
494  TT = Triple(M.getTargetTriple());
495 
496  // Emit the runtime hook even if no counters are present.
497  bool MadeChange = emitRuntimeHook();
498 
499  // Improve compile time by avoiding linear scans when there is no work.
500  GlobalVariable *CoverageNamesVar =
502  if (!containsProfilingIntrinsics(M) && !CoverageNamesVar)
503  return MadeChange;
504 
505  // We did not know how many value sites there would be inside
506  // the instrumented function. This is counting the number of instrumented
507  // target value sites to enter it as field in the profile data variable.
508  for (Function &F : M) {
509  InstrProfIncrementInst *FirstProfIncInst = nullptr;
510  for (BasicBlock &BB : F)
511  for (auto I = BB.begin(), E = BB.end(); I != E; I++)
512  if (auto *Ind = dyn_cast<InstrProfValueProfileInst>(I))
513  computeNumValueSiteCounts(Ind);
514  else if (FirstProfIncInst == nullptr)
515  FirstProfIncInst = dyn_cast<InstrProfIncrementInst>(I);
516 
517  // Value profiling intrinsic lowering requires per-function profile data
518  // variable to be created first.
519  if (FirstProfIncInst != nullptr)
520  static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
521  }
522 
523  for (Function &F : M)
524  MadeChange |= lowerIntrinsics(&F);
525 
526  if (CoverageNamesVar) {
527  lowerCoverageData(CoverageNamesVar);
528  MadeChange = true;
529  }
530 
531  if (!MadeChange)
532  return false;
533 
534  emitVNodes();
535  emitNameData();
536  emitRegistration();
537  emitUses();
538  emitInitialization();
539  return true;
540 }
541 
542 static FunctionCallee
544  bool IsRange = false) {
545  LLVMContext &Ctx = M.getContext();
546  auto *ReturnTy = Type::getVoidTy(M.getContext());
547 
549  if (auto AK = TLI.getExtAttrForI32Param(false))
550  AL = AL.addParamAttribute(M.getContext(), 2, AK);
551 
552  if (!IsRange) {
553  Type *ParamTypes[] = {
554 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
556  };
557  auto *ValueProfilingCallTy =
558  FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
560  ValueProfilingCallTy, AL);
561  } else {
562  Type *RangeParamTypes[] = {
563 #define VALUE_RANGE_PROF 1
564 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
566 #undef VALUE_RANGE_PROF
567  };
568  auto *ValueRangeProfilingCallTy =
569  FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
571  ValueRangeProfilingCallTy, AL);
572  }
573 }
574 
575 void InstrProfiling::computeNumValueSiteCounts(InstrProfValueProfileInst *Ind) {
576  GlobalVariable *Name = Ind->getName();
577  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
578  uint64_t Index = Ind->getIndex()->getZExtValue();
579  auto It = ProfileDataMap.find(Name);
580  if (It == ProfileDataMap.end()) {
581  PerFunctionProfileData PD;
582  PD.NumValueSites[ValueKind] = Index + 1;
583  ProfileDataMap[Name] = PD;
584  } else if (It->second.NumValueSites[ValueKind] <= Index)
585  It->second.NumValueSites[ValueKind] = Index + 1;
586 }
587 
588 void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
589  GlobalVariable *Name = Ind->getName();
590  auto It = ProfileDataMap.find(Name);
591  assert(It != ProfileDataMap.end() && It->second.DataVar &&
592  "value profiling detected in function with no counter incerement");
593 
594  GlobalVariable *DataVar = It->second.DataVar;
595  uint64_t ValueKind = Ind->getValueKind()->getZExtValue();
596  uint64_t Index = Ind->getIndex()->getZExtValue();
597  for (uint32_t Kind = IPVK_First; Kind < ValueKind; ++Kind)
598  Index += It->second.NumValueSites[Kind];
599 
600  IRBuilder<> Builder(Ind);
601  bool IsRange = (Ind->getValueKind()->getZExtValue() ==
602  llvm::InstrProfValueKind::IPVK_MemOPSize);
603  CallInst *Call = nullptr;
604  if (!IsRange) {
605  Value *Args[3] = {Ind->getTargetValue(),
606  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
607  Builder.getInt32(Index)};
608  Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args);
609  } else {
610  Value *Args[6] = {
611  Ind->getTargetValue(),
612  Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
613  Builder.getInt32(Index),
614  Builder.getInt64(MemOPSizeRangeStart),
615  Builder.getInt64(MemOPSizeRangeLast),
616  Builder.getInt64(MemOPSizeLarge == 0 ? INT64_MIN : MemOPSizeLarge)};
617  Call =
618  Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true), Args);
619  }
620  if (auto AK = TLI->getExtAttrForI32Param(false))
621  Call->addParamAttr(2, AK);
622  Ind->replaceAllUsesWith(Call);
623  Ind->eraseFromParent();
624 }
625 
626 void InstrProfiling::lowerIncrement(InstrProfIncrementInst *Inc) {
627  GlobalVariable *Counters = getOrCreateRegionCounters(Inc);
628 
629  IRBuilder<> Builder(Inc);
630  uint64_t Index = Inc->getIndex()->getZExtValue();
631  Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters->getValueType(),
632  Counters, 0, Index);
633 
634  if (Options.Atomic || AtomicCounterUpdateAll) {
635  Builder.CreateAtomicRMW(AtomicRMWInst::Add, Addr, Inc->getStep(),
637  } else {
638  Value *IncStep = Inc->getStep();
639  Value *Load = Builder.CreateLoad(IncStep->getType(), Addr, "pgocount");
640  auto *Count = Builder.CreateAdd(Load, Inc->getStep());
641  auto *Store = Builder.CreateStore(Count, Addr);
642  if (isCounterPromotionEnabled())
643  PromotionCandidates.emplace_back(cast<Instruction>(Load), Store);
644  }
645  Inc->eraseFromParent();
646 }
647 
648 void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) {
649  ConstantArray *Names =
650  cast<ConstantArray>(CoverageNamesVar->getInitializer());
651  for (unsigned I = 0, E = Names->getNumOperands(); I < E; ++I) {
652  Constant *NC = Names->getOperand(I);
653  Value *V = NC->stripPointerCasts();
654  assert(isa<GlobalVariable>(V) && "Missing reference to function name");
655  GlobalVariable *Name = cast<GlobalVariable>(V);
656 
658  ReferencedNames.push_back(Name);
659  NC->dropAllReferences();
660  }
661  CoverageNamesVar->eraseFromParent();
662 }
663 
664 /// Get the name of a profiling variable for a particular function.
666  StringRef NamePrefix = getInstrProfNameVarPrefix();
667  StringRef Name = Inc->getName()->getName().substr(NamePrefix.size());
668  Function *F = Inc->getParent()->getParent();
669  Module *M = F->getParent();
670  if (!DoHashBasedCounterSplit || !isIRPGOFlagSet(M) ||
671  !canRenameComdatFunc(*F))
672  return (Prefix + Name).str();
673  uint64_t FuncHash = Inc->getHash()->getZExtValue();
674  SmallVector<char, 24> HashPostfix;
675  if (Name.endswith((Twine(".") + Twine(FuncHash)).toStringRef(HashPostfix)))
676  return (Prefix + Name).str();
677  return (Prefix + Name + "." + Twine(FuncHash)).str();
678 }
679 
680 static inline bool shouldRecordFunctionAddr(Function *F) {
681  // Check the linkage
682  bool HasAvailableExternallyLinkage = F->hasAvailableExternallyLinkage();
683  if (!F->hasLinkOnceLinkage() && !F->hasLocalLinkage() &&
684  !HasAvailableExternallyLinkage)
685  return true;
686 
687  // A function marked 'alwaysinline' with available_externally linkage can't
688  // have its address taken. Doing so would create an undefined external ref to
689  // the function, which would fail to link.
690  if (HasAvailableExternallyLinkage &&
691  F->hasFnAttribute(Attribute::AlwaysInline))
692  return false;
693 
694  // Prohibit function address recording if the function is both internal and
695  // COMDAT. This avoids the profile data variable referencing internal symbols
696  // in COMDAT.
697  if (F->hasLocalLinkage() && F->hasComdat())
698  return false;
699 
700  // Check uses of this function for other than direct calls or invokes to it.
701  // Inline virtual functions have linkeOnceODR linkage. When a key method
702  // exists, the vtable will only be emitted in the TU where the key method
703  // is defined. In a TU where vtable is not available, the function won't
704  // be 'addresstaken'. If its address is not recorded here, the profile data
705  // with missing address may be picked by the linker leading to missing
706  // indirect call target info.
707  return F->hasAddressTaken() || F->hasLinkOnceLinkage();
708 }
709 
711  // Don't do this for Darwin. compiler-rt uses linker magic.
712  if (TT.isOSDarwin())
713  return false;
714  // Use linker script magic to get data/cnts/name start/end.
715  if (TT.isOSLinux() || TT.isOSFreeBSD() || TT.isOSNetBSD() ||
716  TT.isOSFuchsia() || TT.isPS4CPU() || TT.isOSWindows())
717  return false;
718 
719  return true;
720 }
721 
723 InstrProfiling::getOrCreateRegionCounters(InstrProfIncrementInst *Inc) {
724  GlobalVariable *NamePtr = Inc->getName();
725  auto It = ProfileDataMap.find(NamePtr);
726  PerFunctionProfileData PD;
727  if (It != ProfileDataMap.end()) {
728  if (It->second.RegionCounters)
729  return It->second.RegionCounters;
730  PD = It->second;
731  }
732 
733  // Match the linkage and visibility of the name global, except on COFF, where
734  // the linkage must be local and consequentially the visibility must be
735  // default.
736  Function *Fn = Inc->getParent()->getParent();
737  GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage();
739  if (TT.isOSBinFormatCOFF()) {
741  Visibility = GlobalValue::DefaultVisibility;
742  }
743 
744  // Move the name variable to the right section. Place them in a COMDAT group
745  // if the associated function is a COMDAT. This will make sure that only one
746  // copy of counters of the COMDAT function will be emitted after linking. Keep
747  // in mind that this pass may run before the inliner, so we need to create a
748  // new comdat group for the counters and profiling data. If we use the comdat
749  // of the parent function, that will result in relocations against discarded
750  // sections.
751  Comdat *Cmdt = nullptr;
752  GlobalValue::LinkageTypes CounterLinkage = Linkage;
753  if (needsComdatForCounter(*Fn, *M)) {
754  StringRef CmdtPrefix = getInstrProfComdatPrefix();
755  if (TT.isOSBinFormatCOFF()) {
756  // For COFF, the comdat group name must be the name of a symbol in the
757  // group. Use the counter variable name, and upgrade its linkage to
758  // something externally visible, like linkonce_odr.
759  CmdtPrefix = getInstrProfCountersVarPrefix();
760  CounterLinkage = GlobalValue::LinkOnceODRLinkage;
761  }
762  Cmdt = M->getOrInsertComdat(getVarName(Inc, CmdtPrefix));
763  }
764 
765  uint64_t NumCounters = Inc->getNumCounters()->getZExtValue();
766  LLVMContext &Ctx = M->getContext();
767  ArrayType *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters);
768 
769  // Create the counters variable.
770  auto *CounterPtr =
771  new GlobalVariable(*M, CounterTy, false, Linkage,
772  Constant::getNullValue(CounterTy),
774  CounterPtr->setVisibility(Visibility);
775  CounterPtr->setSection(
776  getInstrProfSectionName(IPSK_cnts, TT.getObjectFormat()));
777  CounterPtr->setAlignment(8);
778  CounterPtr->setComdat(Cmdt);
779  CounterPtr->setLinkage(CounterLinkage);
780 
781  auto *Int8PtrTy = Type::getInt8PtrTy(Ctx);
782  // Allocate statically the array of pointers to value profile nodes for
783  // the current function.
784  Constant *ValuesPtrExpr = ConstantPointerNull::get(Int8PtrTy);
785  if (ValueProfileStaticAlloc && !needsRuntimeRegistrationOfSectionRange(TT)) {
786  uint64_t NS = 0;
787  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
788  NS += PD.NumValueSites[Kind];
789  if (NS) {
790  ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS);
791 
792  auto *ValuesVar =
793  new GlobalVariable(*M, ValuesTy, false, Linkage,
794  Constant::getNullValue(ValuesTy),
796  ValuesVar->setVisibility(Visibility);
797  ValuesVar->setSection(
798  getInstrProfSectionName(IPSK_vals, TT.getObjectFormat()));
799  ValuesVar->setAlignment(8);
800  ValuesVar->setComdat(Cmdt);
801  ValuesPtrExpr =
803  }
804  }
805 
806  // Create data variable.
807  auto *Int16Ty = Type::getInt16Ty(Ctx);
808  auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1);
809  Type *DataTypes[] = {
810 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
812  };
813  auto *DataTy = StructType::get(Ctx, makeArrayRef(DataTypes));
814 
815  Constant *FunctionAddr = shouldRecordFunctionAddr(Fn)
816  ? ConstantExpr::getBitCast(Fn, Int8PtrTy)
817  : ConstantPointerNull::get(Int8PtrTy);
818 
819  Constant *Int16ArrayVals[IPVK_Last + 1];
820  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
821  Int16ArrayVals[Kind] = ConstantInt::get(Int16Ty, PD.NumValueSites[Kind]);
822 
823  Constant *DataVals[] = {
824 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
826  };
827  auto *Data = new GlobalVariable(*M, DataTy, false, Linkage,
828  ConstantStruct::get(DataTy, DataVals),
830  Data->setVisibility(Visibility);
831  Data->setSection(getInstrProfSectionName(IPSK_data, TT.getObjectFormat()));
832  Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
833  Data->setComdat(Cmdt);
834 
835  PD.RegionCounters = CounterPtr;
836  PD.DataVar = Data;
837  ProfileDataMap[NamePtr] = PD;
838 
839  // Mark the data variable as used so that it isn't stripped out.
840  UsedVars.push_back(Data);
841  // Now that the linkage set by the FE has been passed to the data and counter
842  // variables, reset Name variable's linkage and visibility to private so that
843  // it can be removed later by the compiler.
845  // Collect the referenced names to be used by emitNameData.
846  ReferencedNames.push_back(NamePtr);
847 
848  return CounterPtr;
849 }
850 
851 void InstrProfiling::emitVNodes() {
852  if (!ValueProfileStaticAlloc)
853  return;
854 
855  // For now only support this on platforms that do
856  // not require runtime registration to discover
857  // named section start/end.
859  return;
860 
861  size_t TotalNS = 0;
862  for (auto &PD : ProfileDataMap) {
863  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
864  TotalNS += PD.second.NumValueSites[Kind];
865  }
866 
867  if (!TotalNS)
868  return;
869 
870  uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
871 // Heuristic for small programs with very few total value sites.
872 // The default value of vp-counters-per-site is chosen based on
873 // the observation that large apps usually have a low percentage
874 // of value sites that actually have any profile data, and thus
875 // the average number of counters per site is low. For small
876 // apps with very few sites, this may not be true. Bump up the
877 // number of counters in this case.
878 #define INSTR_PROF_MIN_VAL_COUNTS 10
879  if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS)
880  NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2);
881 
882  auto &Ctx = M->getContext();
883  Type *VNodeTypes[] = {
884 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
886  };
887  auto *VNodeTy = StructType::get(Ctx, makeArrayRef(VNodeTypes));
888 
889  ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters);
890  auto *VNodesVar = new GlobalVariable(
891  *M, VNodesTy, false, GlobalValue::PrivateLinkage,
893  VNodesVar->setSection(
894  getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat()));
895  UsedVars.push_back(VNodesVar);
896 }
897 
898 void InstrProfiling::emitNameData() {
899  std::string UncompressedData;
900 
901  if (ReferencedNames.empty())
902  return;
903 
904  std::string CompressedNameStr;
905  if (Error E = collectPGOFuncNameStrings(ReferencedNames, CompressedNameStr,
906  DoNameCompression)) {
907  report_fatal_error(toString(std::move(E)), false);
908  }
909 
910  auto &Ctx = M->getContext();
911  auto *NamesVal = ConstantDataArray::getString(
912  Ctx, StringRef(CompressedNameStr), false);
913  NamesVar = new GlobalVariable(*M, NamesVal->getType(), true,
914  GlobalValue::PrivateLinkage, NamesVal,
916  NamesSize = CompressedNameStr.size();
917  NamesVar->setSection(
918  getInstrProfSectionName(IPSK_name, TT.getObjectFormat()));
919  // On COFF, it's important to reduce the alignment down to 1 to prevent the
920  // linker from inserting padding before the start of the names section or
921  // between names entries.
922  NamesVar->setAlignment(1);
923  UsedVars.push_back(NamesVar);
924 
925  for (auto *NamePtr : ReferencedNames)
926  NamePtr->eraseFromParent();
927 }
928 
929 void InstrProfiling::emitRegistration() {
931  return;
932 
933  // Construct the function.
934  auto *VoidTy = Type::getVoidTy(M->getContext());
935  auto *VoidPtrTy = Type::getInt8PtrTy(M->getContext());
936  auto *Int64Ty = Type::getInt64Ty(M->getContext());
937  auto *RegisterFTy = FunctionType::get(VoidTy, false);
938  auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage,
940  RegisterF->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
941  if (Options.NoRedZone)
942  RegisterF->addFnAttr(Attribute::NoRedZone);
943 
944  auto *RuntimeRegisterTy = FunctionType::get(VoidTy, VoidPtrTy, false);
945  auto *RuntimeRegisterF =
948 
949  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF));
950  for (Value *Data : UsedVars)
951  if (Data != NamesVar && !isa<Function>(Data))
952  IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
953 
954  if (NamesVar) {
955  Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
956  auto *NamesRegisterTy =
957  FunctionType::get(VoidTy, makeArrayRef(ParamTypes), false);
958  auto *NamesRegisterF =
961  IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
962  IRB.getInt64(NamesSize)});
963  }
964 
965  IRB.CreateRetVoid();
966 }
967 
968 bool InstrProfiling::emitRuntimeHook() {
969  // We expect the linker to be invoked with -u<hook_var> flag for linux,
970  // for which case there is no need to emit the user function.
971  if (TT.isOSLinux())
972  return false;
973 
974  // If the module's provided its own runtime, we don't need to do anything.
976  return false;
977 
978  // Declare an external variable that will pull in the runtime initialization.
979  auto *Int32Ty = Type::getInt32Ty(M->getContext());
980  auto *Var =
982  nullptr, getInstrProfRuntimeHookVarName());
983 
984  // Make a function that uses it.
988  User->addFnAttr(Attribute::NoInline);
989  if (Options.NoRedZone)
990  User->addFnAttr(Attribute::NoRedZone);
991  User->setVisibility(GlobalValue::HiddenVisibility);
992  if (TT.supportsCOMDAT())
993  User->setComdat(M->getOrInsertComdat(User->getName()));
994 
996  auto *Load = IRB.CreateLoad(Int32Ty, Var);
997  IRB.CreateRet(Load);
998 
999  // Mark the user variable as used so that it isn't stripped out.
1000  UsedVars.push_back(User);
1001  return true;
1002 }
1003 
1004 void InstrProfiling::emitUses() {
1005  if (!UsedVars.empty())
1006  appendToUsed(*M, UsedVars);
1007 }
1008 
1009 void InstrProfiling::emitInitialization() {
1010  // Create ProfileFileName variable. Don't don't this for the
1011  // context-sensitive instrumentation lowering: This lowering is after
1012  // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should
1013  // have already create the variable before LTO/ThinLTO linking.
1014  if (!IsCS)
1015  createProfileFileNameVar(*M, Options.InstrProfileOutput);
1016  Function *RegisterF = M->getFunction(getInstrProfRegFuncsName());
1017  if (!RegisterF)
1018  return;
1019 
1020  // Create the initialization function.
1021  auto *VoidTy = Type::getVoidTy(M->getContext());
1022  auto *F = Function::Create(FunctionType::get(VoidTy, false),
1025  F->setUnnamedAddr(GlobalValue::UnnamedAddr::Global);
1026  F->addFnAttr(Attribute::NoInline);
1027  if (Options.NoRedZone)
1028  F->addFnAttr(Attribute::NoRedZone);
1029 
1030  // Add the basic block and the necessary calls.
1031  IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F));
1032  IRB.CreateCall(RegisterF, {});
1033  IRB.CreateRetVoid();
1034 
1035  appendToGlobalCtors(*M, F, 0);
1036 }
cl::opt< std::string > MemOPSizeRange("memop-size-range", cl::desc("Set the range of size in memory intrinsic calls to be profiled " "precisely, in a format of <start_val>:<end_val>"), cl::init(""))
bool isOSDarwin() const
isOSDarwin - Is this a "Darwin" OS (OS X, iOS, or watchOS).
Definition: Triple.h:476
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks &#39;this&#39; from the containing basic block and deletes it.
Definition: Instruction.cpp:67
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable...
Definition: InstrProf.h:156
const std::string & getTargetTriple() const
Get the target triple which is a string describing the target host.
Definition: Module.h:240
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:2576
bool hasLocalLinkage() const
Definition: GlobalValue.h:435
Helper class for SSA formation on a set of values defined in multiple blocks.
Definition: SSAUpdater.h:38
Value * getPointerOperand(Value *V)
A helper function that returns the pointer operand of a load, store or GEP instruction.
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
ConstantInt * getIndex() const
void dropAllReferences()
Drop all references to operands.
Definition: User.h:294
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
Definition: InstrProf.h:82
LLVM_NODISCARD bool endswith(StringRef Suffix) const
Check if this string ends with the given Suffix.
Definition: StringRef.h:267
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:769
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
cl::opt< unsigned > MemOPSizeLarge("memop-size-large", cl::desc("Set large value thresthold in memory intrinsic size profiling. " "Value of 0 disables the large value profiling."), cl::init(8192))
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
Definition: InstrProf.h:126
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve &#39;CreateLoad(Ty, Ptr, "...")&#39; correctly, instead of converting the string to &#39;bool...
Definition: IRBuilder.h:1392
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:65
amdgpu Simplify well known AMD library false FunctionCallee Value const Twine & Name
bool hasDedicatedExits() const
Return true if no exit block for the loop has a predecessor that is outside the loop.
Definition: LoopInfoImpl.h:85
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:164
static bool containsProfilingIntrinsics(Module &M)
Check if the module contains uses of any profiling intrinsics.
#define INSTR_PROF_MIN_VAL_COUNTS
This class represents a function call, abstracting a target machine&#39;s calling convention.
bool hasAvailableExternallyLinkage() const
Definition: GlobalValue.h:422
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
Definition: InstrProf.cpp:970
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
Definition: LoopInfoImpl.h:173
const GlobalVariable * getNamedGlobal(StringRef Name) const
Return the global variable in the module with the specified name, of arbitrary type.
Definition: Module.h:405
Like Internal, but omit from symbol table.
Definition: GlobalValue.h:56
Externally visible function.
Definition: GlobalValue.h:48
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:320
GlobalVariable * getGlobalVariable(StringRef Name) const
Look up the specified global variable in the module symbol table.
Definition: Module.h:390
This file provides the interface for LLVM&#39;s PGO Instrumentation lowering pass.
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:164
F(f)
bool isOSFuchsia() const
Definition: Triple.h:496
An instruction for reading from memory.
Definition: Instructions.h:167
static unsigned InstrCount
ELFYAML::ELF_STV Visibility
Definition: ELFYAML.cpp:848
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:176
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:137
static IntegerType * getInt16Ty(LLVMContext &C)
Definition: Type.cpp:174
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
Definition: InstrProf.h:91
static Constant * getNullValue(Type *Ty)
Constructor to create a &#39;0&#39; constant of arbitrary type.
Definition: Constants.cpp:274
Attribute::AttrKind getExtAttrForI32Param(bool Signed=true) const
Returns extension attribute kind to be used for i32 parameters corresponding to C-level int or unsign...
ModulePass * createInstrProfilingLegacyPass(const InstrProfOptions &Options=InstrProfOptions(), bool IsCS=false)
Insert frontend instrumentation based profiling. Parameter IsCS indicates if.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
Definition: PassSupport.h:50
Hexagon Hardware Loops
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
Definition: InstrProf.h:145
std::string toString(Error E)
Write all error messages (if any) in E to a string.
Definition: Error.h:966
StringRef getName(ID id)
Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
Definition: Function.cpp:625
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:694
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Definition: InstrProf.h:85
#define INT64_MIN
Definition: DataTypes.h:80
LLVMContext & getContext() const
Get the global data context.
Definition: Module.h:244
Instrumentation based profiling lowering pass.
ReturnInst * CreateRet(Value *V)
Create a &#39;ret <val>&#39; instruction.
Definition: IRBuilder.h:828
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:742
This file contains the simple types necessary to represent the attributes associated with functions a...
Error collectPGOFuncNameStrings(ArrayRef< std::string > NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
Definition: InstrProf.cpp:378
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1049
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Definition: InstrProf.h:117
LLVM_NODISCARD StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
Definition: StringRef.h:578
static StructType * get(LLVMContext &Context, ArrayRef< Type *> Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
Definition: Type.cpp:341
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Definition: InstrProf.h:137
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1421
void getExitBlocks(SmallVectorImpl< BlockT *> &ExitBlocks) const
Return all of the successor blocks of this loop.
Definition: LoopInfoImpl.h:62
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:266
Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1767
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
void appendToUsed(Module &M, ArrayRef< GlobalValue *> Values)
Adds global values to the llvm.used list.
Class to represent array types.
Definition: DerivedTypes.h:400
VisibilityTypes
An enumeration for the kinds of visibility of global values.
Definition: GlobalValue.h:62
Memory SSA
Definition: MemorySSA.cpp:64
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:130
LinkageTypes getLinkage() const
Definition: GlobalValue.h:450
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:429
bool isOSNetBSD() const
Definition: Triple.h:484
bool hasLinkOnceLinkage() const
Definition: GlobalValue.h:425
GlobalVariable * getName() const
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree...
Definition: Dominators.h:144
Value * getOperand(unsigned i) const
Definition: User.h:169
ConstantInt * getNumCounters() const
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:156
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
Definition: InstrProf.cpp:950
static FunctionCallee getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI, bool IsRange=false)
static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix)
Get the name of a profiling variable for a particular function.
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
Definition: Constants.cpp:1782
bool isOSWindows() const
Tests whether the OS is Windows.
Definition: Triple.h:529
void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, int64_t &RangeLast)
Definition: InstrProf.cpp:993
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Definition: InstrProf.cpp:924
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Same, but only replaced by something equivalent.
Definition: GlobalValue.h:51
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:427
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)
Definition: Function.h:135
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:148
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:153
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:216
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
Definition: Constants.cpp:1410
VisibilityTypes getVisibility() const
Definition: GlobalValue.h:232
ConstantInt * getIndex() const
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1033
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:64
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
Definition: InstrProf.h:72
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This is an important base class in LLVM.
Definition: Constant.h:41
ValueKind
Value kinds.
This file contains the declarations for the subclasses of Constant, which represent the different fla...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:370
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function&#39;s PGO names...
Definition: InstrProf.h:102
void eraseFromParent()
eraseFromParent - This method unlinks &#39;this&#39; from the containing module and deletes it...
Definition: Globals.cpp:358
Represent the analysis usage information of a pass.
static Type * getVoidTy(LLVMContext &C)
Definition: Type.cpp:160
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1192
bool isPS4CPU() const
Tests whether the target is the PS4 CPU.
Definition: Triple.h:639
static bool shouldRecordFunctionAddr(Function *F)
static FunctionType * get(Type *Result, ArrayRef< Type *> Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:296
static Constant * get(StructType *T, ArrayRef< Constant *> V)
Definition: Constants.cpp:1053
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:99
ConstantInt * getInt64(uint64_t C)
Get a constant 64-bit value.
Definition: IRBuilder.h:311
AtomicRMWInst * CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val, AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System)
Definition: IRBuilder.h:1487
void getExitingBlocks(SmallVectorImpl< BlockT *> &ExitingBlocks) const
Return all blocks inside the loop that have successors outside of the loop.
Definition: LoopInfoImpl.h:34
StringRef toStringRef(bool B)
Construct a string ref from a boolean.
Definition: StringExtras.h:52
const Constant * stripPointerCasts() const
Definition: Constant.h:177
Comdat * getOrInsertComdat(StringRef Name)
Return the Comdat in the module with the specified name.
Definition: Module.cpp:482
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:159
size_t size() const
Definition: SmallVector.h:52
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Definition: Type.cpp:219
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:43
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
Definition: InstrProf.h:88
void addParamAttr(unsigned ArgNo, Attribute::AttrKind Kind)
Adds the attribute to the indicated argument.
Definition: InstrTypes.h:1311
unsigned getNumOperands() const
Definition: User.h:191
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements...
Definition: SmallPtrSet.h:417
static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT)
Helper class for promoting a collection of loads and stores into SSA Form using the SSAUpdater...
Definition: SSAUpdater.h:136
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:841
Module.h This file contains the declarations for the Module class.
Provides information about what library functions are available for the current target.
bool isOSLinux() const
Tests whether the OS is Linux.
Definition: Triple.h:577
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:631
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:83
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:301
FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T, AttributeList AttributeList)
Look up the specified function in the module symbol table.
Definition: Module.cpp:143
#define NC
Definition: regutils.h:42
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:444
bool isOSFreeBSD() const
Definition: Triple.h:492
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
Definition: Module.cpp:174
ConstantArray - Constant Array Declarations.
Definition: Constants.h:413
LinkageTypes
An enumeration for the kinds of linkage for global values.
Definition: GlobalValue.h:47
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
Definition: InstrProf.h:150
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Definition: InstrProf.h:132
Options for the frontend instrumentation based profiling pass.
bool hasComdat() const
Definition: GlobalObject.h:99
This represents the llvm.instrprof_increment intrinsic.
ReturnInst * CreateRetVoid()
Create a &#39;ret void&#39; instruction.
Definition: IRBuilder.h:823
Analysis providing branch probability information.
static IntegerType * getInt32Ty(LLVMContext &C)
Definition: Type.cpp:175
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:464
ConstantInt * getHash() const
StringRef getName() const
Return a constant reference to the value&#39;s name.
Definition: Value.cpp:214
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:106
#define I(x, y, z)
Definition: MD5.cpp:58
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:224
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
Definition: Type.cpp:580
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
Type * getValueType() const
Definition: GlobalValue.h:275
Rename collisions when linking (static functions).
Definition: GlobalValue.h:55
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2009
SmallVector< LoopT *, 4 > getLoopsInPreorder()
Return all of the loops in the function in preorder across the loop nests, with siblings in forward p...
Definition: LoopInfoImpl.h:582
Analysis pass providing the TargetLibraryInfo.
bool hasAddressTaken(const User **=nullptr) const
hasAddressTaken - returns true if there are any uses of this function other than direct calls or invo...
Definition: Function.cpp:1257
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
StringRef getInstrProfValueRangeProfFuncName()
Return the name profile runtime entry point to do value range profiling.
Definition: InstrProf.h:77
aarch64 promote const
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:565
LLVM Value Representation.
Definition: Value.h:72
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Definition: InstrProf.h:94
GlobalVariable * getName() const
Lightweight error class with error context and mandatory checking.
Definition: Error.h:157
ConstantInt * getValueKind() const
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
A container for analyses that lazily runs them and caches their results.
StringRef getInstrProfComdatPrefix()
Return the name prefix of the COMDAT group for instrumentation variables associated with a COMDAT fun...
Definition: InstrProf.h:98
static bool lowerIntrinsics(Module &M)
#define LLVM_DEBUG(X)
Definition: Debug.h:122
This represents the llvm.instrprof_value_profile intrinsic.
static InstrProfIncrementInst * castToIncrementInst(Instruction *Instr)
void setSection(StringRef S)
Change the section for this global.
Definition: Globals.cpp:188
bool use_empty() const
Definition: Value.h:322
Optional< uint64_t > getBlockProfileCount(const BasicBlock *BB) const
Returns the estimated profile count of BB.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
IntegerType * Int32Ty
Value * CreateConstInBoundsGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1, const Twine &Name="")
Definition: IRBuilder.h:1648
INITIALIZE_PASS_BEGIN(InstrProfilingLegacyPass, "instrprof", "Frontend instrumentation-based coverage lowering.", false, false) INITIALIZE_PASS_END(InstrProfilingLegacyPass
const BasicBlock * getParent() const
Definition: Instruction.h:66