LLVM  15.0.0git
MemProfiler.cpp
Go to the documentation of this file.
1 //===- MemProfiler.cpp - memory allocation and access profiler ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of MemProfiler. Memory accesses are instrumented
10 // to increment the access count held in a shadow memory location, or
11 // alternatively to call into the runtime. Memory intrinsic calls (memmove,
12 // memcpy, memset) are changed to call the memory profiling runtime version
13 // instead.
14 //
15 //===----------------------------------------------------------------------===//
16 
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/ADT/Triple.h"
23 #include "llvm/IR/Constant.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/IR/Function.h"
26 #include "llvm/IR/GlobalValue.h"
27 #include "llvm/IR/IRBuilder.h"
28 #include "llvm/IR/Instruction.h"
29 #include "llvm/IR/IntrinsicInst.h"
30 #include "llvm/IR/Module.h"
31 #include "llvm/IR/Type.h"
32 #include "llvm/IR/Value.h"
33 #include "llvm/InitializePasses.h"
34 #include "llvm/Pass.h"
37 #include "llvm/Support/Debug.h"
40 
41 using namespace llvm;
42 
43 #define DEBUG_TYPE "memprof"
44 
45 constexpr int LLVM_MEM_PROFILER_VERSION = 1;
46 
47 // Size of memory mapped to a single shadow location.
49 
50 // Scale from granularity down to shadow size.
52 
53 constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
55 // On Emscripten, the system needs more than one priorities for constructors.
57 constexpr char MemProfInitName[] = "__memprof_init";
59  "__memprof_version_mismatch_check_v";
60 
62  "__memprof_shadow_memory_dynamic_address";
63 
64 constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
65 
66 // Command-line flags.
67 
69  "memprof-guard-against-version-mismatch",
70  cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
71  cl::init(true));
72 
73 // This flag may need to be replaced with -f[no-]memprof-reads.
74 static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
75  cl::desc("instrument read instructions"),
76  cl::Hidden, cl::init(true));
77 
78 static cl::opt<bool>
79  ClInstrumentWrites("memprof-instrument-writes",
80  cl::desc("instrument write instructions"), cl::Hidden,
81  cl::init(true));
82 
84  "memprof-instrument-atomics",
85  cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
86  cl::init(true));
87 
89  "memprof-use-callbacks",
90  cl::desc("Use callbacks instead of inline instrumentation sequences."),
91  cl::Hidden, cl::init(false));
92 
94  ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
95  cl::desc("Prefix for memory access callbacks"),
96  cl::Hidden, cl::init("__memprof_"));
97 
98 // These flags allow to change the shadow mapping.
99 // The shadow mapping looks like
100 // Shadow = ((Mem & mask) >> scale) + offset
101 
102 static cl::opt<int> ClMappingScale("memprof-mapping-scale",
103  cl::desc("scale of memprof shadow mapping"),
105 
106 static cl::opt<int>
107  ClMappingGranularity("memprof-mapping-granularity",
108  cl::desc("granularity of memprof shadow mapping"),
110 
111 static cl::opt<bool> ClStack("memprof-instrument-stack",
112  cl::desc("Instrument scalar stack variables"),
113  cl::Hidden, cl::init(false));
114 
115 // Debug flags.
116 
117 static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
118  cl::init(0));
119 
120 static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
121  cl::desc("Debug func"));
122 
123 static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
124  cl::Hidden, cl::init(-1));
125 
126 static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
127  cl::Hidden, cl::init(-1));
128 
129 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
130 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
131 STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
132 STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
133 
134 namespace {
135 
136 /// This struct defines the shadow mapping using the rule:
137 /// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
138 struct ShadowMapping {
139  ShadowMapping() {
140  Scale = ClMappingScale;
141  Granularity = ClMappingGranularity;
142  Mask = ~(Granularity - 1);
143  }
144 
145  int Scale;
146  int Granularity;
147  uint64_t Mask; // Computed as ~(Granularity-1)
148 };
149 
150 static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
153 }
154 
155 struct InterestingMemoryAccess {
156  Value *Addr = nullptr;
157  bool IsWrite;
158  Type *AccessTy;
160  Value *MaybeMask = nullptr;
161 };
162 
163 /// Instrument the code in module to profile memory accesses.
164 class MemProfiler {
165 public:
166  MemProfiler(Module &M) {
167  C = &(M.getContext());
168  LongSize = M.getDataLayout().getPointerSizeInBits();
169  IntptrTy = Type::getIntNTy(*C, LongSize);
170  }
171 
172  /// If it is an interesting memory access, populate information
173  /// about the access and return a InterestingMemoryAccess struct.
174  /// Otherwise return None.
176  isInterestingMemoryAccess(Instruction *I) const;
177 
178  void instrumentMop(Instruction *I, const DataLayout &DL,
179  InterestingMemoryAccess &Access);
180  void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
181  Value *Addr, uint32_t TypeSize, bool IsWrite);
183  Instruction *I, Value *Addr, Type *AccessTy,
184  bool IsWrite);
185  void instrumentMemIntrinsic(MemIntrinsic *MI);
186  Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
187  bool instrumentFunction(Function &F);
188  bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
189  bool insertDynamicShadowAtFunctionEntry(Function &F);
190 
191 private:
192  void initializeCallbacks(Module &M);
193 
194  LLVMContext *C;
195  int LongSize;
196  Type *IntptrTy;
197  ShadowMapping Mapping;
198 
199  // These arrays is indexed by AccessIsWrite
200  FunctionCallee MemProfMemoryAccessCallback[2];
201  FunctionCallee MemProfMemoryAccessCallbackSized[2];
202 
203  FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
204  Value *DynamicShadowOffset = nullptr;
205 };
206 
207 class MemProfilerLegacyPass : public FunctionPass {
208 public:
209  static char ID;
210 
211  explicit MemProfilerLegacyPass() : FunctionPass(ID) {
213  }
214 
215  StringRef getPassName() const override { return "MemProfilerFunctionPass"; }
216 
217  bool runOnFunction(Function &F) override {
218  MemProfiler Profiler(*F.getParent());
219  return Profiler.instrumentFunction(F);
220  }
221 };
222 
223 class ModuleMemProfiler {
224 public:
225  ModuleMemProfiler(Module &M) { TargetTriple = Triple(M.getTargetTriple()); }
226 
227  bool instrumentModule(Module &);
228 
229 private:
230  Triple TargetTriple;
231  ShadowMapping Mapping;
232  Function *MemProfCtorFunction = nullptr;
233 };
234 
235 class ModuleMemProfilerLegacyPass : public ModulePass {
236 public:
237  static char ID;
238 
239  explicit ModuleMemProfilerLegacyPass() : ModulePass(ID) {
241  }
242 
243  StringRef getPassName() const override { return "ModuleMemProfiler"; }
244 
245  void getAnalysisUsage(AnalysisUsage &AU) const override {}
246 
247  bool runOnModule(Module &M) override {
248  ModuleMemProfiler MemProfiler(M);
249  return MemProfiler.instrumentModule(M);
250  }
251 };
252 
253 } // end anonymous namespace
254 
256 
259  Module &M = *F.getParent();
260  MemProfiler Profiler(M);
261  if (Profiler.instrumentFunction(F))
262  return PreservedAnalyses::none();
263  return PreservedAnalyses::all();
264 }
265 
267 
270  ModuleMemProfiler Profiler(M);
271  if (Profiler.instrumentModule(M))
272  return PreservedAnalyses::none();
273  return PreservedAnalyses::all();
274 }
275 
277 
278 INITIALIZE_PASS_BEGIN(MemProfilerLegacyPass, "memprof",
279  "MemProfiler: profile memory allocations and accesses.",
280  false, false)
281 INITIALIZE_PASS_END(MemProfilerLegacyPass, "memprof",
282  "MemProfiler: profile memory allocations and accesses.",
284 
286  return new MemProfilerLegacyPass();
287 }
288 
290 
291 INITIALIZE_PASS(ModuleMemProfilerLegacyPass, "memprof-module",
292  "MemProfiler: profile memory allocations and accesses."
293  "ModulePass",
294  false, false)
295 
297  return new ModuleMemProfilerLegacyPass();
298 }
299 
300 Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
301  // (Shadow & mask) >> scale
302  Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
303  Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
304  // (Shadow >> scale) | offset
305  assert(DynamicShadowOffset);
306  return IRB.CreateAdd(Shadow, DynamicShadowOffset);
307 }
308 
309 // Instrument memset/memmove/memcpy
310 void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
311  IRBuilder<> IRB(MI);
312  if (isa<MemTransferInst>(MI)) {
313  IRB.CreateCall(
314  isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
315  {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
316  IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()),
317  IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
318  } else if (isa<MemSetInst>(MI)) {
319  IRB.CreateCall(
320  MemProfMemset,
321  {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()),
322  IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
323  IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
324  }
325  MI->eraseFromParent();
326 }
327 
329 MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
330  // Do not instrument the load fetching the dynamic shadow address.
331  if (DynamicShadowOffset == I)
332  return None;
333 
334  InterestingMemoryAccess Access;
335 
336  if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
337  if (!ClInstrumentReads)
338  return None;
339  Access.IsWrite = false;
340  Access.AccessTy = LI->getType();
341  Access.Addr = LI->getPointerOperand();
342  } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
343  if (!ClInstrumentWrites)
344  return None;
345  Access.IsWrite = true;
346  Access.AccessTy = SI->getValueOperand()->getType();
347  Access.Addr = SI->getPointerOperand();
348  } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
349  if (!ClInstrumentAtomics)
350  return None;
351  Access.IsWrite = true;
352  Access.AccessTy = RMW->getValOperand()->getType();
353  Access.Addr = RMW->getPointerOperand();
354  } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
355  if (!ClInstrumentAtomics)
356  return None;
357  Access.IsWrite = true;
358  Access.AccessTy = XCHG->getCompareOperand()->getType();
359  Access.Addr = XCHG->getPointerOperand();
360  } else if (auto *CI = dyn_cast<CallInst>(I)) {
361  auto *F = CI->getCalledFunction();
362  if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
363  F->getIntrinsicID() == Intrinsic::masked_store)) {
364  unsigned OpOffset = 0;
365  if (F->getIntrinsicID() == Intrinsic::masked_store) {
366  if (!ClInstrumentWrites)
367  return None;
368  // Masked store has an initial operand for the value.
369  OpOffset = 1;
370  Access.AccessTy = CI->getArgOperand(0)->getType();
371  Access.IsWrite = true;
372  } else {
373  if (!ClInstrumentReads)
374  return None;
375  Access.AccessTy = CI->getType();
376  Access.IsWrite = false;
377  }
378 
379  auto *BasePtr = CI->getOperand(0 + OpOffset);
380  Access.MaybeMask = CI->getOperand(2 + OpOffset);
381  Access.Addr = BasePtr;
382  }
383  }
384 
385  if (!Access.Addr)
386  return None;
387 
388  // Do not instrument acesses from different address spaces; we cannot deal
389  // with them.
390  Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
391  if (PtrTy->getPointerAddressSpace() != 0)
392  return None;
393 
394  // Ignore swifterror addresses.
395  // swifterror memory addresses are mem2reg promoted by instruction
396  // selection. As such they cannot have regular uses like an instrumentation
397  // function and it makes no sense to track them as memory.
398  if (Access.Addr->isSwiftError())
399  return None;
400 
401  // Peel off GEPs and BitCasts.
402  auto *Addr = Access.Addr->stripInBoundsOffsets();
403 
404  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
405  // Do not instrument PGO counter updates.
406  if (GV->hasSection()) {
407  StringRef SectionName = GV->getSection();
408  // Check if the global is in the PGO counters section.
409  auto OF = Triple(I->getModule()->getTargetTriple()).getObjectFormat();
410  if (SectionName.endswith(
411  getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
412  return None;
413  }
414 
415  // Do not instrument accesses to LLVM internal variables.
416  if (GV->getName().startswith("__llvm"))
417  return None;
418  }
419 
420  const DataLayout &DL = I->getModule()->getDataLayout();
421  Access.TypeSize = DL.getTypeStoreSizeInBits(Access.AccessTy);
422  return Access;
423 }
424 
426  Instruction *I, Value *Addr,
427  Type *AccessTy, bool IsWrite) {
428  auto *VTy = cast<FixedVectorType>(AccessTy);
429  uint64_t ElemTypeSize = DL.getTypeStoreSizeInBits(VTy->getScalarType());
430  unsigned Num = VTy->getNumElements();
431  auto *Zero = ConstantInt::get(IntptrTy, 0);
432  for (unsigned Idx = 0; Idx < Num; ++Idx) {
433  Value *InstrumentedAddress = nullptr;
434  Instruction *InsertBefore = I;
435  if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
436  // dyn_cast as we might get UndefValue
437  if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
438  if (Masked->isZero())
439  // Mask is constant false, so no instrumentation needed.
440  continue;
441  // If we have a true or undef value, fall through to instrumentAddress.
442  // with InsertBefore == I
443  }
444  } else {
445  IRBuilder<> IRB(I);
446  Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
447  Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
448  InsertBefore = ThenTerm;
449  }
450 
451  IRBuilder<> IRB(InsertBefore);
452  InstrumentedAddress =
453  IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
454  instrumentAddress(I, InsertBefore, InstrumentedAddress, ElemTypeSize,
455  IsWrite);
456  }
457 }
458 
459 void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
460  InterestingMemoryAccess &Access) {
461  // Skip instrumentation of stack accesses unless requested.
462  if (!ClStack && isa<AllocaInst>(getUnderlyingObject(Access.Addr))) {
463  if (Access.IsWrite)
464  ++NumSkippedStackWrites;
465  else
466  ++NumSkippedStackReads;
467  return;
468  }
469 
470  if (Access.IsWrite)
471  NumInstrumentedWrites++;
472  else
473  NumInstrumentedReads++;
474 
475  if (Access.MaybeMask) {
476  instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
477  Access.AccessTy, Access.IsWrite);
478  } else {
479  // Since the access counts will be accumulated across the entire allocation,
480  // we only update the shadow access count for the first location and thus
481  // don't need to worry about alignment and type size.
482  instrumentAddress(I, I, Access.Addr, Access.TypeSize, Access.IsWrite);
483  }
484 }
485 
486 void MemProfiler::instrumentAddress(Instruction *OrigIns,
487  Instruction *InsertBefore, Value *Addr,
488  uint32_t TypeSize, bool IsWrite) {
489  IRBuilder<> IRB(InsertBefore);
490  Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
491 
492  if (ClUseCalls) {
493  IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
494  return;
495  }
496 
497  // Create an inline sequence to compute shadow location, and increment the
498  // value by one.
499  Type *ShadowTy = Type::getInt64Ty(*C);
500  Type *ShadowPtrTy = PointerType::get(ShadowTy, 0);
501  Value *ShadowPtr = memToShadow(AddrLong, IRB);
502  Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
503  Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
505  ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
506  IRB.CreateStore(ShadowValue, ShadowAddr);
507 }
508 
509 // Create the variable for the profile file name.
511  const MDString *MemProfFilename =
512  dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
513  if (!MemProfFilename)
514  return;
515  assert(!MemProfFilename->getString().empty() &&
516  "Unexpected MemProfProfileFilename metadata with empty string");
517  Constant *ProfileNameConst = ConstantDataArray::getString(
518  M.getContext(), MemProfFilename->getString(), true);
519  GlobalVariable *ProfileNameVar = new GlobalVariable(
520  M, ProfileNameConst->getType(), /*isConstant=*/true,
522  Triple TT(M.getTargetTriple());
523  if (TT.supportsCOMDAT()) {
524  ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
525  ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
526  }
527 }
528 
529 bool ModuleMemProfiler::instrumentModule(Module &M) {
530  // Create a module constructor.
531  std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
532  std::string VersionCheckName =
534  : "";
535  std::tie(MemProfCtorFunction, std::ignore) =
537  MemProfInitName, /*InitArgTypes=*/{},
538  /*InitArgs=*/{}, VersionCheckName);
539 
540  const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
541  appendToGlobalCtors(M, MemProfCtorFunction, Priority);
542 
544 
545  return true;
546 }
547 
548 void MemProfiler::initializeCallbacks(Module &M) {
549  IRBuilder<> IRB(*C);
550 
551  for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
552  const std::string TypeStr = AccessIsWrite ? "store" : "load";
553 
554  SmallVector<Type *, 3> Args2 = {IntptrTy, IntptrTy};
555  SmallVector<Type *, 2> Args1{1, IntptrTy};
556  MemProfMemoryAccessCallbackSized[AccessIsWrite] =
557  M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr + "N",
558  FunctionType::get(IRB.getVoidTy(), Args2, false));
559 
560  MemProfMemoryAccessCallback[AccessIsWrite] =
561  M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + TypeStr,
562  FunctionType::get(IRB.getVoidTy(), Args1, false));
563  }
564  MemProfMemmove = M.getOrInsertFunction(
565  ClMemoryAccessCallbackPrefix + "memmove", IRB.getInt8PtrTy(),
566  IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), IntptrTy);
567  MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
568  IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
569  IRB.getInt8PtrTy(), IntptrTy);
570  MemProfMemset = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset",
571  IRB.getInt8PtrTy(), IRB.getInt8PtrTy(),
572  IRB.getInt32Ty(), IntptrTy);
573 }
574 
575 bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
576  // For each NSObject descendant having a +load method, this method is invoked
577  // by the ObjC runtime before any of the static constructors is called.
578  // Therefore we need to instrument such methods with a call to __memprof_init
579  // at the beginning in order to initialize our runtime before any access to
580  // the shadow memory.
581  // We cannot just ignore these methods, because they may call other
582  // instrumented functions.
583  if (F.getName().find(" load]") != std::string::npos) {
584  FunctionCallee MemProfInitFunction =
586  IRBuilder<> IRB(&F.front(), F.front().begin());
587  IRB.CreateCall(MemProfInitFunction, {});
588  return true;
589  }
590  return false;
591 }
592 
593 bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
594  IRBuilder<> IRB(&F.front().front());
595  Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
597  if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
598  cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
599  DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
600  return true;
601 }
602 
603 bool MemProfiler::instrumentFunction(Function &F) {
604  if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
605  return false;
606  if (ClDebugFunc == F.getName())
607  return false;
608  if (F.getName().startswith("__memprof_"))
609  return false;
610 
611  bool FunctionModified = false;
612 
613  // If needed, insert __memprof_init.
614  // This function needs to be called even if the function body is not
615  // instrumented.
616  if (maybeInsertMemProfInitAtFunctionEntry(F))
617  FunctionModified = true;
618 
619  LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
620 
621  initializeCallbacks(*F.getParent());
622 
623  SmallVector<Instruction *, 16> ToInstrument;
624 
625  // Fill the set of memory operations to instrument.
626  for (auto &BB : F) {
627  for (auto &Inst : BB) {
628  if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
629  ToInstrument.push_back(&Inst);
630  }
631  }
632 
633  if (ToInstrument.empty()) {
634  LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
635  << " " << F << "\n");
636 
637  return FunctionModified;
638  }
639 
640  FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
641 
642  int NumInstrumented = 0;
643  for (auto *Inst : ToInstrument) {
644  if (ClDebugMin < 0 || ClDebugMax < 0 ||
645  (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
647  isInterestingMemoryAccess(Inst);
648  if (Access)
649  instrumentMop(Inst, F.getParent()->getDataLayout(), *Access);
650  else
651  instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
652  }
653  NumInstrumented++;
654  }
655 
656  if (NumInstrumented > 0)
657  FunctionModified = true;
658 
659  LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
660  << F << "\n");
661 
662  return FunctionModified;
663 }
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:152
llvm::IRBuilderBase::CreateIntCast
Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")
Definition: IRBuilder.h:2021
llvm::initializeMemProfilerLegacyPassPass
void initializeMemProfilerLegacyPassPass(PassRegistry &)
llvm::IRBuilderBase::getInt32Ty
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:501
llvm::IRBuilderBase::CreateStore
StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)
Definition: IRBuilder.h:1662
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:104
llvm::GlobalObject::setComdat
void setComdat(Comdat *C)
Definition: Globals.cpp:194
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
ClDebugFunc
static cl::opt< std::string > ClDebugFunc("memprof-debug-func", cl::Hidden, cl::desc("Debug func"))
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
ClInstrumentWrites
static cl::opt< bool > ClInstrumentWrites("memprof-instrument-writes", cl::desc("instrument write instructions"), cl::Hidden, cl::init(true))
llvm::ModulePass
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
Definition: Pass.h:248
IntrinsicInst.h
llvm::Function
Definition: Function.h:60
StringRef.h
Pass.h
llvm::PointerType::get
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Definition: Type.cpp:727
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
Statistic.h
llvm::Type::getPointerAddressSpace
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
Definition: DerivedTypes.h:729
llvm::IRBuilder<>
llvm::GlobalVariable
Definition: GlobalVariable.h:39
llvm::FunctionType::get
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
Definition: Type.cpp:361
ValueTracking.h
llvm::Triple
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:139
DefaultShadowGranularity
constexpr uint64_t DefaultShadowGranularity
Definition: MemProfiler.cpp:48
llvm::PreservedAnalyses::none
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
Definition: PassManager.h:155
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Module.h
DefaultShadowScale
constexpr uint64_t DefaultShadowScale
Definition: MemProfiler.cpp:51
llvm::MemIntrinsic
This is the common base class for memset/memcpy/memmove.
Definition: IntrinsicInst.h:957
LLVM_MEM_PROFILER_VERSION
constexpr int LLVM_MEM_PROFILER_VERSION
Definition: MemProfiler.cpp:45
llvm::Optional
Definition: APInt.h:33
Vector
So we should use XX3Form_Rcr to implement intrinsic Convert DP outs ins xscvdpsp No builtin are required Round &Convert QP DP(dword[1] is set to zero) No builtin are required Round to Quad Precision because you need to assign rounding mode in instruction Provide builtin(set f128:$vT,(int_ppc_vsx_xsrqpi f128:$vB))(set f128 yields< n x< ty > >< result > yields< ty >< result > No builtin are required Load Store Vector
Definition: README_P9.txt:497
ClDebugMax
static cl::opt< int > ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"), cl::Hidden, cl::init(-1))
llvm::createProfileFileNameVar
void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput)
Definition: InstrProf.cpp:1182
and
We currently generate a but we really shouldn eax ecx xorl edx divl ecx eax divl ecx movl eax ret A similar code sequence works for division We currently compile i32 v2 eax eax jo LBB1_2 and
Definition: README.txt:1271
llvm::Triple::isOSEmscripten
bool isOSEmscripten() const
Tests whether the OS is Emscripten.
Definition: Triple.h:639
llvm::IRBuilderBase::CreateIntToPtr
Value * CreateIntToPtr(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1947
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::createMemProfilerFunctionPass
FunctionPass * createMemProfilerFunctionPass()
Definition: MemProfiler.cpp:285
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
Instruction.h
CommandLine.h
GlobalValue.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
llvm::ModuleMemProfilerPass::ModuleMemProfilerPass
ModuleMemProfilerPass()
llvm::IRBuilderBase::CreateGEP
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1725
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::IRBuilderBase::CreateLoad
LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)
Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...
Definition: IRBuilder.h:1649
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
MemProfInitName
constexpr char MemProfInitName[]
Definition: MemProfiler.cpp:57
false
Definition: StackSlotColoring.cpp:141
llvm::MemProfilerPass::MemProfilerPass
MemProfilerPass()
llvm::Instruction
Definition: Instruction.h:42
InstrProf.h
llvm::Triple::getObjectFormat
ObjectFormatType getObjectFormat() const
Get the object format for this triple.
Definition: Triple.h:371
llvm::STATISTIC
STATISTIC(NumFunctions, "Total number of functions")
llvm::codeview::EncodedFramePtrReg::BasePtr
@ BasePtr
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:926
llvm::getUnderlyingObject
const Value * getUnderlyingObject(const Value *V, unsigned MaxLookup=6)
This method strips off any GEP address adjustments and pointer casts from the specified value,...
Definition: ValueTracking.cpp:4450
ClInsertVersionCheck
static cl::opt< bool > ClInsertVersionCheck("memprof-guard-against-version-mismatch", cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden, cl::init(true))
ClDebug
static cl::opt< int > ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden, cl::init(0))
ClDebugMin
static cl::opt< int > ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"), cl::Hidden, cl::init(-1))
llvm::None
const NoneType None
Definition: None.h:24
instrumentMaskedLoadOrStore
static void instrumentMaskedLoadOrStore(AddressSanitizer *Pass, const DataLayout &DL, Type *IntptrTy, Value *Mask, Instruction *I, Value *Addr, MaybeAlign Alignment, unsigned Granularity, Type *OpType, bool IsWrite, Value *SizeArgument, bool UseCalls, uint32_t Exp)
Definition: AddressSanitizer.cpp:1393
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
Type.h
llvm::IRBuilderBase::CreateAnd
Value * CreateAnd(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:1348
llvm::IRBuilderBase::CreatePointerCast
Value * CreatePointerCast(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1998
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(MemProfilerLegacyPass, "memprof", "MemProfiler: profile memory allocations and accesses.", false, false) INITIALIZE_PASS_END(MemProfilerLegacyPass
llvm::cl::opt< bool >
MemProfModuleCtorName
constexpr char MemProfModuleCtorName[]
Definition: MemProfiler.cpp:53
MemProfVersionCheckNamePrefix
constexpr char MemProfVersionCheckNamePrefix[]
Definition: MemProfiler.cpp:58
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:297
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::StringRef::empty
constexpr LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:153
llvm::createSanitizerCtorAndInitFunctions
std::pair< Function *, FunctionCallee > createSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, StringRef VersionCheckName=StringRef())
Creates sanitizer constructor function, and calls sanitizer's init function from it.
Definition: ModuleUtils.cpp:135
uint64_t
ClStack
static cl::opt< bool > ClStack("memprof-instrument-stack", cl::desc("Instrument scalar stack variables"), cl::Hidden, cl::init(false))
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:78
llvm::GlobalValue::WeakAnyLinkage
@ WeakAnyLinkage
Keep one copy of named function when linking (weak)
Definition: GlobalValue.h:52
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
llvm::declareSanitizerInitFunction
FunctionCallee declareSanitizerInitFunction(Module &M, StringRef InitName, ArrayRef< Type * > InitArgTypes)
Definition: ModuleUtils.cpp:114
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::PICLevel::NotPIC
@ NotPIC
Definition: CodeGen.h:33
llvm::IRBuilderBase::getInt8PtrTy
PointerType * getInt8PtrTy(unsigned AddrSpace=0)
Fetch the type representing a pointer to an 8-bit integer value.
Definition: IRBuilder.h:549
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
ClUseCalls
static cl::opt< bool > ClUseCalls("memprof-use-callbacks", cl::desc("Use callbacks instead of inline instrumentation sequences."), cl::Hidden, cl::init(false))
MemProfEmscriptenCtorAndDtorPriority
constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority
Definition: MemProfiler.cpp:56
llvm::GlobalValue::setLinkage
void setLinkage(LinkageTypes LT)
Definition: GlobalValue.h:502
llvm::IRBuilderBase::CreateAdd
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1200
ClInstrumentReads
static cl::opt< bool > ClInstrumentReads("memprof-instrument-reads", cl::desc("instrument read instructions"), cl::Hidden, cl::init(true))
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::IRBuilderBase::CreateExtractElement
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2276
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Triple.h
DataLayout.h
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
uint32_t
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::ConstantDataArray::getString
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
Definition: Constants.cpp:3065
llvm::GlobalValue::AvailableExternallyLinkage
@ AvailableExternallyLinkage
Available for inspection, not emission.
Definition: GlobalValue.h:49
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:173
llvm::AtomicRMWInst
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:714
runOnFunction
static bool runOnFunction(Function &F, bool PostInlining)
Definition: EntryExitInstrumenter.cpp:69
Constant.h
profile
sample profile
Definition: SampleProfile.cpp:1836
llvm::Type::getInt64Ty
static IntegerType * getInt64Ty(LLVMContext &C)
Definition: Type.cpp:240
llvm::createModuleMemProfilerLegacyPassPass
ModulePass * createModuleMemProfilerLegacyPassPass()
llvm::Type::getIntNTy
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition: Type.cpp:243
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:158
llvm::initializeModuleMemProfilerLegacyPassPass
void initializeModuleMemProfilerLegacyPassPass(PassRegistry &)
llvm::SectionName
Definition: DWARFSection.h:21
MemProfFilenameVar
constexpr char MemProfFilenameVar[]
Definition: MemProfiler.cpp:64
llvm::TypeSize
Definition: TypeSize.h:435
llvm::MemProfilerPass::run
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Definition: MemProfiler.cpp:257
Function.h
llvm::ModuleMemProfilerPass::run
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Definition: MemProfiler.cpp:268
INITIALIZE_PASS
INITIALIZE_PASS(ModuleMemProfilerLegacyPass, "memprof-module", "MemProfiler: profile memory allocations and accesses." "ModulePass", false, false) ModulePass *llvm
Definition: MemProfiler.cpp:291
ClMappingGranularity
static cl::opt< int > ClMappingGranularity("memprof-mapping-granularity", cl::desc("granularity of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowGranularity))
MemProfCtorAndDtorPriority
constexpr uint64_t MemProfCtorAndDtorPriority
Definition: MemProfiler.cpp:54
MemProfShadowMemoryDynamicAddress
constexpr char MemProfShadowMemoryDynamicAddress[]
Definition: MemProfiler.cpp:61
ClMappingScale
static cl::opt< int > ClMappingScale("memprof-mapping-scale", cl::desc("scale of memprof shadow mapping"), cl::Hidden, cl::init(DefaultShadowScale))
llvm::getInstrProfSectionName
std::string getInstrProfSectionName(InstrProfSectKind IPSK, Triple::ObjectFormatType OF, bool AddSegmentInfo=true)
Return the name of the profile section corresponding to IPSK.
Definition: InstrProf.cpp:214
llvm::GlobalValue::ExternalLinkage
@ ExternalLinkage
Externally visible function.
Definition: GlobalValue.h:48
memprof
memprof
Definition: MemProfiler.cpp:281
SmallVector.h
llvm::IRBuilderBase::CreateLShr
Value * CreateLShr(Value *LHS, Value *RHS, const Twine &Name="", bool isExact=false)
Definition: IRBuilder.h:1310
ModuleUtils.h
ClInstrumentAtomics
static cl::opt< bool > ClInstrumentAtomics("memprof-instrument-atomics", cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, cl::init(true))
llvm::to_string
std::string to_string(const T &Value)
Definition: ScopedPrinter.h:85
MemProfiler.h
llvm::FunctionCallee
A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...
Definition: DerivedTypes.h:165
ClMemoryAccessCallbackPrefix
static cl::opt< std::string > ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix", cl::desc("Prefix for memory access callbacks"), cl::Hidden, cl::init("__memprof_"))
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:42
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::IRBuilderBase::getVoidTy
Type * getVoidTy()
Fetch the type representing void.
Definition: IRBuilder.h:539
llvm::SplitBlockAndInsertIfThen
Instruction * SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights, DominatorTree *DT, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
Definition: BasicBlockUtils.cpp:1446
llvm::appendToGlobalCtors
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
Definition: ModuleUtils.cpp:65
llvm::cl::desc
Definition: CommandLine.h:405
llvm::MDString::getString
StringRef getString() const
Definition: Metadata.cpp:507
BasicBlockUtils.h
llvm::MDString
A single uniqued string.
Definition: Metadata.h:612
Value.h
InitializePasses.h
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::AtomicCmpXchgInst
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:509
Debug.h
llvm::IRBuilderBase::CreateCall
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2229
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38