27 #define DEBUG_TYPE "instrprof"
32 cl::desc(
"Enable name string compression"),
36 "hash-based-counter-split",
37 cl::desc(
"Rename counter variable of a comdat function based on cfg hash"),
42 cl::desc(
"Do static counter allocation for value profiler"),
45 "vp-counters-per-site",
46 cl::desc(
"The average number of profile counters allocated "
47 "per value profiling site."),
54 class InstrProfilingLegacyPass :
public ModulePass {
59 InstrProfilingLegacyPass() :
ModulePass(
ID), InstrProf() {}
63 return "Frontend instrumentation-based coverage lowering";
66 bool runOnModule(
Module &M)
override {
67 return InstrProf.run(M, getAnalysis<TargetLibraryInfoWrapperPass>().getTLI());
88 InstrProfilingLegacyPass,
"instrprof",
89 "Frontend instrumentation-based coverage lowering.",
false,
false)
97 return new InstrProfilingLegacyPass(Options);
100 bool InstrProfiling::isMachO()
const {
105 StringRef InstrProfiling::getCountersSection()
const {
110 StringRef InstrProfiling::getNameSection()
const {
115 StringRef InstrProfiling::getDataSection()
const {
120 StringRef InstrProfiling::getCoverageSection()
const {
132 bool MadeChange =
false;
138 ProfileDataMap.clear();
147 for (
auto I = BB.begin(),
E = BB.end();
I !=
E;
I++)
148 if (
auto *Ind = dyn_cast<InstrProfValueProfileInst>(
I))
149 computeNumValueSiteCounts(Ind);
150 else if (FirstProfIncInst ==
nullptr)
155 if (FirstProfIncInst !=
nullptr)
156 static_cast<void>(getOrCreateRegionCounters(FirstProfIncInst));
161 for (
auto I = BB.begin(),
E = BB.end();
I !=
E;) {
167 }
else if (
auto *Ind = dyn_cast<InstrProfValueProfileInst>(Instr)) {
168 lowerValueProfileInst(Ind);
175 lowerCoverageData(CoverageNamesVar);
187 emitInitialization();
195 Type *ParamTypes[] = {
196 #define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
199 auto *ValueProfilingCallTy =
202 ValueProfilingCallTy);
203 if (
Function *FunRes = dyn_cast<Function>(Res)) {
205 FunRes->addAttribute(3, AK);
215 auto It = ProfileDataMap.find(Name);
216 if (It == ProfileDataMap.end()) {
217 PerFunctionProfileData
PD;
218 PD.NumValueSites[ValueKind] = Index + 1;
219 ProfileDataMap[
Name] =
PD;
220 }
else if (It->second.NumValueSites[ValueKind] <= Index)
221 It->second.NumValueSites[ValueKind] = Index + 1;
227 auto It = ProfileDataMap.find(Name);
228 assert(It != ProfileDataMap.end() && It->second.DataVar &&
229 "value profiling detected in function with no counter incerement");
235 Index += It->second.NumValueSites[
Kind];
239 Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
240 Builder.getInt32(Index)};
254 Value *Addr = Builder.CreateConstInBoundsGEP2_64(Counters, 0, Index);
255 Value *Count = Builder.CreateLoad(Addr,
"pgocount");
256 Count = Builder.CreateAdd(Count, Inc->
getStep());
261 void InstrProfiling::lowerCoverageData(
GlobalVariable *CoverageNamesVar) {
268 assert(isa<GlobalVariable>(V) &&
"Missing reference to function name");
272 ReferencedNames.push_back(Name);
284 return (Prefix + Name).str();
288 return (Prefix + Name).str();
289 return (Prefix + Name +
"." +
Twine(FuncHash)).str();
344 auto It = ProfileDataMap.find(NamePtr);
345 PerFunctionProfileData
PD;
346 if (It != ProfileDataMap.end()) {
347 if (It->second.RegionCounters)
348 return It->second.RegionCounters;
357 Comdat *ProfileVarsComdat =
nullptr;
370 CounterPtr->setSection(getCountersSection());
371 CounterPtr->setAlignment(8);
372 CounterPtr->setComdat(ProfileVarsComdat);
382 NS += PD.NumValueSites[
Kind];
392 ValuesVar->setAlignment(8);
393 ValuesVar->setComdat(ProfileVarsComdat);
402 Type *DataTypes[] = {
403 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) LLVMType,
412 Constant *Int16ArrayVals[IPVK_Last + 1];
417 #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Init,
424 Data->setSection(getDataSection());
425 Data->setAlignment(INSTR_PROF_DATA_ALIGNMENT);
426 Data->setComdat(ProfileVarsComdat);
428 PD.RegionCounters = CounterPtr;
430 ProfileDataMap[NamePtr] =
PD;
433 UsedVars.push_back(Data);
439 ReferencedNames.push_back(NamePtr);
444 void InstrProfiling::emitVNodes() {
445 if (!ValueProfileStaticAlloc)
455 for (
auto &PD : ProfileDataMap) {
457 TotalNS += PD.second.NumValueSites[
Kind];
463 uint64_t NumCounters = TotalNS * NumCountersPerValueSite;
471 #define INSTR_PROF_MIN_VAL_COUNTS 10
476 Type *VNodeTypes[] = {
477 #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType,
487 UsedVars.push_back(VNodesVar);
490 void InstrProfiling::emitNameData() {
491 std::string UncompressedData;
493 if (ReferencedNames.empty())
496 std::string CompressedNameStr;
498 DoNameCompression)) {
504 Ctx,
StringRef(CompressedNameStr),
false);
508 NamesSize = CompressedNameStr.size();
510 UsedVars.push_back(NamesVar);
513 void InstrProfiling::emitRegistration() {
526 RegisterF->addFnAttr(Attribute::NoRedZone);
529 auto *RuntimeRegisterF =
534 for (
Value *Data : UsedVars)
535 if (Data != NamesVar)
536 IRB.CreateCall(RuntimeRegisterF, IRB.CreateBitCast(Data, VoidPtrTy));
539 Type *ParamTypes[] = {VoidPtrTy, Int64Ty};
540 auto *NamesRegisterTy =
542 auto *NamesRegisterF =
545 IRB.CreateCall(NamesRegisterF, {IRB.CreateBitCast(NamesVar, VoidPtrTy),
546 IRB.getInt64(NamesSize)});
552 void InstrProfiling::emitRuntimeHook() {
573 User->addFnAttr(Attribute::NoInline);
575 User->addFnAttr(Attribute::NoRedZone);
581 auto *
Load = IRB.CreateLoad(Var);
585 UsedVars.push_back(
User);
588 void InstrProfiling::emitUses() {
589 if (!UsedVars.empty())
593 void InstrProfiling::emitInitialization() {
596 if (!InstrProfileOutput.
empty()) {
602 ProfileNameConst, INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR));
604 if (TT.supportsCOMDAT()) {
607 StringRef(INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_NAME_VAR))));
621 F->addFnAttr(Attribute::NoInline);
623 F->addFnAttr(Attribute::NoRedZone);
628 IRB.CreateCall(RegisterF, {});
void setVisibility(VisibilityTypes V)
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
StringRef getInstrProfRuntimeHookVarUseFuncName()
Return the name of the compiler generated function that references the runtime hook variable...
LinkageTypes getLinkage() const
static Constant * getString(LLVMContext &Context, StringRef Initializer, bool AddNull=true)
This method constructs a CDS and initializes it with a text string.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool endswith(StringRef Suffix) const
Check if this string ends with the given Suffix.
VisibilityTypes getVisibility() const
StringRef getInstrProfNameVarPrefix()
Return the name prefix of variables containing instrumented function names.
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
StringRef getInstrProfRegFuncsName()
Return the name of function that registers all the per-function control data at program startup time ...
A Module instance is used to store all the information related to an LLVM module. ...
unsigned getNumOperands() const
#define INSTR_PROF_MIN_VAL_COUNTS
This class represents a function call, abstracting a target machine's calling convention.
bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken=false)
Check if we can safely rename this Comdat function.
ConstantInt * getIndex() const
Like Internal, but omit from symbol table.
Externally visible function.
bool hasAvailableExternallyLinkage() const
const Function * getParent() const
Return the enclosing method, or null if none.
This file provides the interface for LLVM's PGO Instrumentation lowering pass.
static IntegerType * getInt64Ty(LLVMContext &C)
ModulePass * createInstrProfilingLegacyPass(const InstrProfOptions &Options=InstrProfOptions())
Insert frontend instrumentation based profiling.
static IntegerType * getInt16Ty(LLVMContext &C)
const Constant * getInitializer() const
getInitializer - Return the initializer for this global variable.
StringRef getInstrProfValuesVarPrefix()
Return the name prefix of value profile variables.
const std::string & getTargetTriple() const
Get the target triple which is a string describing the target host.
ConstantInt * getHash() const
static Constant * getNullValue(Type *Ty)
Constructor to create a '0' constant of arbitrary type.
StringRef getName() const
Return a constant reference to the value's name.
StringRef getInstrProfDataSectionName(bool AddSegment)
Return the name of the data section containing per-function control data.
AnalysisUsage & addRequired()
#define INITIALIZE_PASS_DEPENDENCY(depName)
StringRef getInstrProfInitFuncName()
Return the name of the runtime initialization method that is generated by the compiler.
std::string toString(Error E)
Write all error messages (if any) in E to a string.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Frontend instrumentation based coverage lowering
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
StringRef getInstrProfDataVarPrefix()
Return the name prefix of variables containing per-function control data.
Instrumentation based profiling lowering pass.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
ConstantInt * getIndex() const
StringRef getInstrProfCoverageSectionName(bool AddSegment)
Return the name of the section containing function coverage mapping data.
StringRef getCoverageUnusedNamesVarName()
Return the name of the internal variable recording the array of PGO name vars referenced by the cover...
Error collectPGOFuncNameStrings(const std::vector< std::string > &NameStrs, bool doCompression, std::string &Result)
Given a vector of strings (function PGO names) NameStrs, the method generates a combined string Resul...
ConstantInt * getNumCounters() const
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
StringRef getInstrProfNamesRegFuncName()
Return the name of the runtime interface that registers the PGO name strings.
Class to represent array types.
static FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)
This static method is the primary way of constructing a FunctionType.
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Attribute::AttrKind getExtAttrForI32Param(bool Signed=true) const
Returns extension attribute kind to be used for i32 parameters corresponding to C-level int or unsign...
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const
size - Get the string size.
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
bool isIRPGOFlagSet(const Module *M)
Check if INSTR_PROF_RAW_VERSION_VAR is defined.
static GCRegistry::Add< CoreCLRGC > E("coreclr","CoreCLR-compatible GC")
Function * getFunction(StringRef Name) const
Look up the specified function in the module symbol table.
static std::string getVarName(InstrProfIncrementInst *Inc, StringRef Prefix)
Get the name of a profiling variable for a particular function.
static Constant * getBitCast(Constant *C, Type *Ty, bool OnlyIfReduced=false)
GlobalVariable * getName() const
bool needsComdatForCounter(const Function &F, const Module &M)
Check if we can use Comdat for profile variables.
Same, but only replaced by something equivalent.
initializer< Ty > init(const Ty &Val)
A set of analyses that are preserved following a run of a transformation pass.
* if(!EatIfPresent(lltok::kw_thread_local)) return false
ParseOptionalThreadLocal := /*empty.
Constant * stripPointerCasts()
static ConstantPointerNull * get(PointerType *T)
Static factory methods - Return objects of the specified value.
GlobalVariable * getName() const
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs...ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Constant * getOrInsertFunction(StringRef Name, FunctionType *T, AttributeSet AttributeList)
Look up the specified function in the module symbol table.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
This is an important class for using LLVM in a threaded context.
StringRef getInstrProfValueProfFuncName()
Return the name profile runtime entry point to do value profiling for a given site.
This is an important base class in LLVM.
StringRef getInstrProfNamesVarName()
Return the name of the variable holding the strings (possibly compressed) of all function's PGO names...
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef substr(size_t Start, size_t N=npos) const
Return a reference to the substring from [Start, Start + N).
StringRef getInstrProfValuesSectionName(bool AddSegment)
Return the name of data section containing pointers to value profile counters/nodes.
Represent the analysis usage information of a pass.
static Type * getVoidTy(LLVMContext &C)
static bool shouldRecordFunctionAddr(Function *F)
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE,"Assign register bank of generic virtual registers", false, false) RegBankSelect
Value * getOperand(unsigned i) const
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Value * getTargetValue() const
Frontend instrumentation based coverage false
static Constant * get(StructType *T, ArrayRef< Constant * > V)
INITIALIZE_PASS_BEGIN(InstrProfilingLegacyPass,"instrprof","Frontend instrumentation-based coverage lowering.", false, false) INITIALIZE_PASS_END(InstrProfilingLegacyPass
Comdat * getOrInsertComdat(StringRef Name)
Return the Comdat in the module with the specified name.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
std::string InstrProfileOutput
static bool needsRuntimeRegistrationOfSectionRange(const Module &M)
static PointerType * getInt8PtrTy(LLVMContext &C, unsigned AS=0)
Triple - Helper class for working with autoconf configuration names.
StringRef getInstrProfCountersVarPrefix()
Return the name prefix of profile counter variables.
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
Module.h This file contains the declarations for the Module class.
Type * getType() const
All values are typed, get the type of this value.
Provides information about what library functions are available for the current target.
A constant pointer value that points to null.
pgo instr PGO instrumentation
StringRef getInstrProfVNodesSectionName(bool AddSegment)
Return the name of data section containing nodes holdling value profiling data.
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)
Append F to the list of global ctors of module M with the given Priority.
void setPreservesCFG()
This function should be called by the pass, iff they do not:
void setLinkage(LinkageTypes LT)
ConstantArray - Constant Array Declarations.
StringRef getInstrProfRuntimeHookVarName()
Return the name of the hook variable defined in profile runtime library.
StringRef getInstrProfRegFuncName()
Return the name of the runtime interface that registers per-function control data for one instrumente...
Options for the frontend instrumentation based profiling pass.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const
empty - Check if the string is empty.
static StringRef toStringRef(bool B)
Construct a string ref from a boolean.
This represents the llvm.instrprof_increment intrinsic.
bool hasLinkOnceLinkage() const
bool hasAddressTaken(const User **=nullptr) const
hasAddressTaken - returns true if there are any uses of this function other than direct calls or invo...
StringRef getInstrProfCountersSectionName(bool AddSegment)
Return the name of data section containing profile counter variables.
static IntegerType * getInt32Ty(LLVMContext &C)
ModulePass class - This class is used to implement unstructured interprocedural optimizations and ana...
static ArrayType * get(Type *ElementType, uint64_t NumElements)
This static method is the primary way to construct an ArrayType.
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Keep one copy of named function when linking (weak)
Rename collisions when linking (static functions).
bool hasLocalLinkage() const
Analysis pass providing the TargetLibraryInfo.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Module * getParent()
Get the module that this global value is contained inside of...
LLVM Value Representation.
void appendToUsed(Module &M, ArrayRef< GlobalValue * > Values)
Adds global values to the llvm.used list.
StringRef getInstrProfVNodesVarName()
Return the name of value profile node array variables:
Lightweight error class with error context and mandatory checking.
static Comdat * getOrCreateProfileComdat(Module &M, Function &F, InstrProfIncrementInst *Inc)
StringRef - Represent a constant reference to a string, i.e.
A container for analyses that lazily runs them and caches their results.
static Function * Create(FunctionType *Ty, LinkageTypes Linkage, const Twine &N="", Module *M=nullptr)
StringRef getInstrProfComdatPrefix()
Return the name prefix of the COMDAT group for instrumentation variables associated with a COMDAT fun...
void addAttribute(unsigned i, Attribute::AttrKind Kind)
adds the attribute to the list of attributes.
ConstantInt * getValueKind() const
StringRef getInstrProfNameSectionName(bool AddSegment)
Return the name of data section containing names of instrumented functions.
This represents the llvm.instrprof_value_profile intrinsic.
static InstrProfIncrementInst * castToIncrementInst(Instruction *Instr)
void setSection(StringRef S)
Change the section for this global.
const BasicBlock * getParent() const
GlobalVariable * getGlobalVariable(StringRef Name) const
Look up the specified global variable in the module symbol table.
static Constant * getOrInsertValueProfilingCall(Module &M, const TargetLibraryInfo &TLI)
LLVMContext & getContext() const
Get the global data context.