#include "llvm/Transforms/Instrumentation/PGOInstrumentation.h"#include "InstrumentationOptions.h"#include "ValueProfileCollector.h"#include "llvm/ADT/APInt.h"#include "llvm/ADT/ArrayRef.h"#include "llvm/ADT/STLExtras.h"#include "llvm/ADT/SmallVector.h"#include "llvm/ADT/Statistic.h"#include "llvm/ADT/StringRef.h"#include "llvm/ADT/StringSet.h"#include "llvm/ADT/Twine.h"#include "llvm/ADT/iterator.h"#include "llvm/ADT/iterator_range.h"#include "llvm/Analysis/BlockFrequencyInfo.h"#include "llvm/Analysis/BranchProbabilityInfo.h"#include "llvm/Analysis/CFG.h"#include "llvm/Analysis/LoopInfo.h"#include "llvm/Analysis/OptimizationRemarkEmitter.h"#include "llvm/Analysis/ProfileSummaryInfo.h"#include "llvm/Analysis/TargetLibraryInfo.h"#include "llvm/Analysis/ValueTracking.h"#include "llvm/IR/Attributes.h"#include "llvm/IR/BasicBlock.h"#include "llvm/IR/CFG.h"#include "llvm/IR/Comdat.h"#include "llvm/IR/Constant.h"#include "llvm/IR/Constants.h"#include "llvm/IR/CycleInfo.h"#include "llvm/IR/DiagnosticInfo.h"#include "llvm/IR/EHPersonalities.h"#include "llvm/IR/Function.h"#include "llvm/IR/GlobalAlias.h"#include "llvm/IR/GlobalValue.h"#include "llvm/IR/GlobalVariable.h"#include "llvm/IR/IRBuilder.h"#include "llvm/IR/InstVisitor.h"#include "llvm/IR/InstrTypes.h"#include "llvm/IR/Instruction.h"#include "llvm/IR/Instructions.h"#include "llvm/IR/IntrinsicInst.h"#include "llvm/IR/Intrinsics.h"#include "llvm/IR/LLVMContext.h"#include "llvm/IR/MDBuilder.h"#include "llvm/IR/Module.h"#include "llvm/IR/PassManager.h"#include "llvm/IR/ProfDataUtils.h"#include "llvm/IR/ProfileSummary.h"#include "llvm/IR/Type.h"#include "llvm/IR/Value.h"#include "llvm/ProfileData/InstrProf.h"#include "llvm/ProfileData/InstrProfReader.h"#include "llvm/Support/BranchProbability.h"#include "llvm/Support/CRC.h"#include "llvm/Support/Casting.h"#include "llvm/Support/CommandLine.h"#include "llvm/Support/Compiler.h"#include "llvm/Support/DOTGraphTraits.h"#include "llvm/Support/Debug.h"#include "llvm/Support/Error.h"#include "llvm/Support/ErrorHandling.h"#include "llvm/Support/GraphWriter.h"#include "llvm/Support/VirtualFileSystem.h"#include "llvm/Support/raw_ostream.h"#include "llvm/TargetParser/Triple.h"#include "llvm/Transforms/Instrumentation/BlockCoverageInference.h"#include "llvm/Transforms/Instrumentation/CFGMST.h"#include "llvm/Transforms/Utils/BasicBlockUtils.h"#include "llvm/Transforms/Utils/Instrumentation.h"#include "llvm/Transforms/Utils/MisExpect.h"#include "llvm/Transforms/Utils/ModuleUtils.h"#include <algorithm>#include <cassert>#include <cstdint>#include <memory>#include <numeric>#include <optional>#include <stack>#include <string>#include <unordered_map>#include <utility>#include <vector>#include "llvm/ProfileData/InstrProfData.inc"Go to the source code of this file.
Classes | |
| struct | llvm::GraphTraits< PGOUseFunc * > |
| struct | llvm::DOTGraphTraits< PGOUseFunc * > |
Namespaces | |
| namespace | llvm |
| This is an optimization pass for GlobalISel generic memory operations. | |
Macros | |
| #define | DEBUG_TYPE "pgo-instrumentation" |
| #define | VALUE_PROF_KIND(Enumerator, Value, Descr) |
| #define | INSTR_PROF_VISIBILITY |
| #define | INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) |
| #define | INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) |
| #define | INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) |
| #define | INSTR_PROF_GPU_SECT(Type, LLVMType, Name, Initializer) |
| #define | INSTR_PROF_RAW_HEADER(Type, Name, Initializer) |
| #define | VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) |
| #define | INSTR_PROF_COMMA |
| #define | INSTR_PROF_DATA_DEFINED |
| #define | COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) |
| #define | COVMAP_HEADER(Type, LLVMType, Name, Initializer) |
| #define | COVINIT_FUNC(Type, LLVMType, Name, Initializer) |
Typedefs | |
| using | VPCandidateInfo = ValueProfileCollector::CandidateInfo |
Functions | |
| STATISTIC (NumOfPGOInstrument, "Number of edges instrumented.") | |
| STATISTIC (NumOfPGOSelectInsts, "Number of select instruction instrumented.") | |
| STATISTIC (NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented.") | |
| STATISTIC (NumOfPGOEdge, "Number of edges.") | |
| STATISTIC (NumOfPGOBB, "Number of basic-blocks.") | |
| STATISTIC (NumOfPGOSplit, "Number of critical edge splits.") | |
| STATISTIC (NumOfPGOFunc, "Number of functions having valid profile counts.") | |
| STATISTIC (NumOfPGOMismatch, "Number of functions having mismatch profile.") | |
| STATISTIC (NumOfPGOMissing, "Number of functions without profile.") | |
| STATISTIC (NumOfPGOICall, "Number of indirect call value instrumentations.") | |
| STATISTIC (NumOfCSPGOInstrument, "Number of edges instrumented in CSPGO.") | |
| STATISTIC (NumOfCSPGOSelectInsts, "Number of select instruction instrumented in CSPGO.") | |
| STATISTIC (NumOfCSPGOMemIntrinsics, "Number of mem intrinsics instrumented in CSPGO.") | |
| STATISTIC (NumOfCSPGOEdge, "Number of edges in CSPGO.") | |
| STATISTIC (NumOfCSPGOBB, "Number of basic-blocks in CSPGO.") | |
| STATISTIC (NumOfCSPGOSplit, "Number of critical edge splits in CSPGO.") | |
| STATISTIC (NumOfCSPGOFunc, "Number of functions having valid profile counts in CSPGO.") | |
| STATISTIC (NumOfCSPGOMismatch, "Number of functions having mismatch profile in CSPGO.") | |
| STATISTIC (NumOfCSPGOMissing, "Number of functions without profile in CSPGO.") | |
| STATISTIC (NumCoveredBlocks, "Number of basic blocks that were executed") | |
| cl::opt< bool > | llvm::NoPGOWarnMismatch ("no-pgo-warn-mismatch", cl::init(false), cl::Hidden, cl::desc("Use this option to turn off/on " "warnings about profile cfg mismatch.")) |
| static std::string | getBranchCondString (Instruction *TI) |
| static GlobalVariable * | createIRLevelProfileFlagVar (const InstrumentationOptions &Opts, Module &M, PGOInstrumentationType InstrumentationType) |
| static bool | canRenameComdat (const InstrumentationOptions &Opts, Function &F, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers) |
| static void | populateEHOperandBundle (VPCandidateInfo &Cand, DenseMap< BasicBlock *, ColorVector > &BlockColors, SmallVectorImpl< OperandBundleDef > &OpBundles) |
| static uint64_t | sumEdgeCount (const ArrayRef< PGOUseEdge * > Edges) |
| static void | setupBBInfoEdges (const FuncPGOInstrumentation< PGOUseEdge, PGOUseBBInfo > &FuncInfo) |
| Set up InEdges/OutEdges for all BBs in the MST. | |
| static void | annotateFunctionWithHashMismatch (Function &F, LLVMContext &ctx) |
| static bool | isIndirectBrTarget (BasicBlock *BB) |
| static uint32_t | getMaxNumAnnotations (const InstrumentationOptions &Opts, InstrProfValueKind ValueProfKind) |
| static void | collectComdatMembers (const InstrumentationOptions &Opts, Module &M, std::unordered_multimap< Comdat *, GlobalValue * > &ComdatMembers) |
| static bool | skipPGOUse (const InstrumentationOptions &Opts, const Function &F) |
| static bool | skipPGOGen (const InstrumentationOptions &Opts, const Function &F) |
| static bool | instrumentAllFunctions (const InstrumentationOptions &Opts, Module &M, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, function_ref< LoopInfo *(Function &)> LookupLI, PGOInstrumentationType InstrumentationType) |
| static void | fixFuncEntryCount (PGOUseFunc &Func, CycleInfo &CI, BranchProbabilityInfo &NBPI) |
| static void | verifyFuncBFI (const InstrumentationOptions &Opts, PGOUseFunc &Func, CycleInfo &CI, BranchProbabilityInfo &NBPI, uint64_t HotCountThreshold, uint64_t ColdCountThreshold) |
| static bool | annotateAllFunctions (const InstrumentationOptions &Opts, Module &M, StringRef ProfileFileName, StringRef ProfileRemappingFileName, vfs::FileSystem &FS, function_ref< TargetLibraryInfo &(Function &)> LookupTLI, function_ref< BranchProbabilityInfo *(Function &)> LookupBPI, function_ref< BlockFrequencyInfo *(Function &)> LookupBFI, function_ref< LoopInfo *(Function &)> LookupLI, ProfileSummaryInfo *PSI, bool IsCS) |
| static std::string | getSimpleNodeName (const BasicBlock *Node) |
| LLVM_ABI void | llvm::setIrrLoopHeaderMetadata (Module *M, Instruction *TI, uint64_t Count) |
Variables | |
| static const char * | ValueProfKindDescr [] |
Macro Definition Documentation
◆ COVINIT_FUNC
| #define COVINIT_FUNC | ( | Type, | |
| LLVMType, | |||
| Name, | |||
| Initializer ) |
◆ COVMAP_FUNC_RECORD
| #define COVMAP_FUNC_RECORD | ( | Type, | |
| LLVMType, | |||
| Name, | |||
| Initializer ) |
◆ COVMAP_HEADER
| #define COVMAP_HEADER | ( | Type, | |
| LLVMType, | |||
| Name, | |||
| Initializer ) |
◆ DEBUG_TYPE
| #define DEBUG_TYPE "pgo-instrumentation" |
Definition at line 135 of file PGOInstrumentation.cpp.
◆ INSTR_PROF_COMMA
| #define INSTR_PROF_COMMA |
◆ INSTR_PROF_DATA
| #define INSTR_PROF_DATA | ( | Type, | |
| LLVMType, | |||
| Name, | |||
| Initializer ) |
◆ INSTR_PROF_DATA_DEFINED
| #define INSTR_PROF_DATA_DEFINED |
◆ INSTR_PROF_GPU_SECT
| #define INSTR_PROF_GPU_SECT | ( | Type, | |
| LLVMType, | |||
| Name, | |||
| Initializer ) |
◆ INSTR_PROF_RAW_HEADER
| #define INSTR_PROF_RAW_HEADER | ( | Type, | |
| Name, | |||
| Initializer ) |
◆ INSTR_PROF_VALUE_NODE
| #define INSTR_PROF_VALUE_NODE | ( | Type, | |
| LLVMType, | |||
| Name, | |||
| Initializer ) |
◆ INSTR_PROF_VISIBILITY
| #define INSTR_PROF_VISIBILITY |
◆ INSTR_PROF_VTABLE_DATA
| #define INSTR_PROF_VTABLE_DATA | ( | Type, | |
| LLVMType, | |||
| Name, | |||
| Initializer ) |
◆ VALUE_PROF_FUNC_PARAM
| #define VALUE_PROF_FUNC_PARAM | ( | ArgType, | |
| ArgName, | |||
| ArgLLVMType ) |
◆ VALUE_PROF_KIND
| #define VALUE_PROF_KIND | ( | Enumerator, | |
| Value, | |||
| Descr ) |
Typedef Documentation
◆ VPCandidateInfo
Definition at line 133 of file PGOInstrumentation.cpp.
Function Documentation
◆ annotateAllFunctions()
|
static |
Definition at line 2053 of file PGOInstrumentation.cpp.
References ColdCountThreshold, collectComdatMembers(), llvm::GenericCycleInfo< ContextT >::compute(), llvm::IndexedInstrProfReader::create(), llvm::StringRef::data(), llvm::dbgs(), llvm::DS_Warning, E(), F, fixFuncEntryCount(), llvm::ProfileSummaryInfo::getOrCompColdCountThreshold(), llvm::ProfileSummaryInfo::getOrCompHotCountThreshold(), llvm::handleAllErrors(), LLVM_DEBUG, llvm::ErrorInfoBase::message(), Msg, llvm::InstrProfRecord::NotPseudo, llvm::PGOVCT_Graph, llvm::PGOVCT_None, llvm::PGOVCT_Text, llvm::PGOViewCounts, llvm::InstrProfRecord::PseudoHot, llvm::ProfileSummary::PSK_CSInstr, llvm::ProfileSummary::PSK_Instr, llvm::ProfileSummaryInfo::refresh(), skipPGOUse(), llvm::SplitIndirectBrCriticalEdges(), uint64_t, llvm::valueOr(), verifyFuncBFI(), llvm::ViewBlockFreqFuncName, llvm::ViewGraph(), and llvm::WriteGraph().
Referenced by llvm::PGOInstrumentationUse::run().
◆ annotateFunctionWithHashMismatch()
|
static |
Definition at line 1246 of file PGOInstrumentation.cpp.
References llvm::cast(), llvm::MDBuilder::createString(), F, llvm::MDTuple::get(), N, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
◆ canRenameComdat()
|
static |
Definition at line 594 of file PGOInstrumentation.cpp.
References assert(), C(), llvm::canRenameComdatFunc(), llvm::dyn_cast(), F, llvm::isa(), and llvm::make_range().
◆ collectComdatMembers()
|
static |
Definition at line 1793 of file PGOInstrumentation.cpp.
Referenced by annotateAllFunctions(), and instrumentAllFunctions().
◆ createIRLevelProfileFlagVar()
|
static |
Definition at line 284 of file PGOInstrumentation.cpp.
References llvm::CSFDO, llvm::CTXPROF, llvm::InstrProfCorrelator::DEBUG_INFO, llvm::GlobalValue::ExternalLinkage, llvm::Type::getInt64Ty(), llvm::Constant::getIntegerValue(), llvm::GlobalValue::HiddenVisibility, INSTR_PROF_QUOTE, INSTR_PROF_RAW_VERSION, INSTR_PROF_RAW_VERSION_VAR, uint64_t, llvm::valueOr(), VARIANT_MASK_BYTE_COVERAGE, VARIANT_MASK_CSIR_PROF, VARIANT_MASK_DBG_CORRELATE, VARIANT_MASK_FUNCTION_ENTRY_ONLY, VARIANT_MASK_INSTR_ENTRY, VARIANT_MASK_INSTR_LOOP_ENTRIES, VARIANT_MASK_IR_PROF, VARIANT_MASK_TEMPORAL_PROF, and llvm::GlobalValue::WeakAnyLinkage.
Referenced by instrumentAllFunctions(), and llvm::PGOInstrumentationGenCreateVar::run().
◆ fixFuncEntryCount()
|
static |
Definition at line 1932 of file PGOInstrumentation.cpp.
References assert(), llvm::APFloatBase::cmpEqual, llvm::APFloatBase::cmpGreaterThan, llvm::dbgs(), F, FuncEntryCount, llvm::BlockFrequencyInfo::getBlockProfileCount(), llvm::APFloat::getZero(), llvm::APFloatBase::IEEEdouble(), LLVM_DEBUG, llvm::APFloatBase::rmNearestTiesToEven, and uint64_t.
Referenced by annotateAllFunctions().
◆ getBranchCondString()
|
static |
Definition at line 246 of file PGOInstrumentation.cpp.
References Cond, llvm::dyn_cast(), llvm::CondBrInst::getCondition(), llvm::User::getOperand(), llvm::CmpInst::getPredicate(), llvm::Value::getType(), llvm::ConstantInt::isMinusOne(), llvm::ConstantInt::isOne(), llvm::ConstantInt::isZero(), llvm::Type::print(), and RHS.
Referenced by llvm::setProfMetadata().
◆ getMaxNumAnnotations()
|
static |
Definition at line 1728 of file PGOInstrumentation.cpp.
References llvm::MaxNumVTableAnnotations.
◆ getSimpleNodeName()
|
static |
Definition at line 2293 of file PGOInstrumentation.cpp.
References Node::printAsOperand().
Referenced by llvm::DOTGraphTraits< PGOUseFunc * >::getNodeLabel().
◆ instrumentAllFunctions()
|
static |
Definition at line 1853 of file PGOInstrumentation.cpp.
References collectComdatMembers(), createIRLevelProfileFlagVar(), llvm::LLVMContext::diagnose(), llvm::DS_Warning, llvm::EnableVTableValueProfiling, F, llvm::FDO, and skipPGOGen().
Referenced by llvm::PGOInstrumentationGen::run().
◆ isIndirectBrTarget()
|
static |
Definition at line 1603 of file PGOInstrumentation.cpp.
References llvm::isa(), and llvm::predecessors().
◆ populateEHOperandBundle()
|
static |
Definition at line 741 of file PGOInstrumentation.cpp.
References llvm::ValueProfileCollector::CandidateInfo::AnnotatedInst, assert(), llvm::dyn_cast(), llvm::SmallVectorImpl< T >::emplace_back(), llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::empty(), llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::find(), llvm::TinyPtrVector< EltTy >::front(), llvm::BasicBlock::getFirstNonPHIIt(), llvm::isa(), llvm::LLVMContext::OB_funclet, and llvm::TinyPtrVector< EltTy >::size().
◆ setupBBInfoEdges()
|
static |
Set up InEdges/OutEdges for all BBs in the MST.
Definition at line 1128 of file PGOInstrumentation.cpp.
References E().
◆ skipPGOGen()
Definition at line 1834 of file PGOInstrumentation.cpp.
References F, and skipPGOUse().
Referenced by instrumentAllFunctions().
◆ skipPGOUse()
Definition at line 1810 of file PGOInstrumentation.cpp.
References llvm::dbgs(), E(), F, llvm::Instruction::getNumSuccessors(), llvm::BasicBlock::getTerminator(), I, llvm::isCriticalEdge(), and LLVM_DEBUG.
Referenced by annotateAllFunctions(), and skipPGOGen().
◆ STATISTIC() [1/20]
| STATISTIC | ( | NumCoveredBlocks | , |
| "Number of basic blocks that were executed" | ) |
◆ STATISTIC() [2/20]
| STATISTIC | ( | NumOfCSPGOBB | , |
| "Number of basic-blocks in CSPGO." | ) |
◆ STATISTIC() [3/20]
| STATISTIC | ( | NumOfCSPGOEdge | , |
| "Number of edges in CSPGO." | ) |
◆ STATISTIC() [4/20]
| STATISTIC | ( | NumOfCSPGOFunc | , |
| "Number of functions having valid profile counts in CSPGO." | ) |
◆ STATISTIC() [5/20]
| STATISTIC | ( | NumOfCSPGOInstrument | , |
| "Number of edges instrumented in CSPGO." | ) |
◆ STATISTIC() [6/20]
| STATISTIC | ( | NumOfCSPGOMemIntrinsics | , |
| "Number of mem intrinsics instrumented in CSPGO." | ) |
◆ STATISTIC() [7/20]
| STATISTIC | ( | NumOfCSPGOMismatch | , |
| "Number of functions having mismatch profile in CSPGO." | ) |
◆ STATISTIC() [8/20]
| STATISTIC | ( | NumOfCSPGOMissing | , |
| "Number of functions without profile in CSPGO." | ) |
◆ STATISTIC() [9/20]
| STATISTIC | ( | NumOfCSPGOSelectInsts | , |
| "Number of select instruction instrumented in CSPGO." | ) |
◆ STATISTIC() [10/20]
| STATISTIC | ( | NumOfCSPGOSplit | , |
| "Number of critical edge splits in CSPGO." | ) |
◆ STATISTIC() [11/20]
| STATISTIC | ( | NumOfPGOBB | , |
| "Number of basic-blocks." | ) |
◆ STATISTIC() [12/20]
| STATISTIC | ( | NumOfPGOEdge | , |
| "Number of edges." | ) |
◆ STATISTIC() [13/20]
| STATISTIC | ( | NumOfPGOFunc | , |
| "Number of functions having valid profile counts." | ) |
◆ STATISTIC() [14/20]
| STATISTIC | ( | NumOfPGOICall | , |
| "Number of indirect call value instrumentations." | ) |
◆ STATISTIC() [15/20]
| STATISTIC | ( | NumOfPGOInstrument | , |
| "Number of edges instrumented." | ) |
◆ STATISTIC() [16/20]
| STATISTIC | ( | NumOfPGOMemIntrinsics | , |
| "Number of mem intrinsics instrumented." | ) |
◆ STATISTIC() [17/20]
| STATISTIC | ( | NumOfPGOMismatch | , |
| "Number of functions having mismatch profile." | ) |
◆ STATISTIC() [18/20]
| STATISTIC | ( | NumOfPGOMissing | , |
| "Number of functions without profile." | ) |
◆ STATISTIC() [19/20]
| STATISTIC | ( | NumOfPGOSelectInsts | , |
| "Number of select instruction instrumented." | ) |
◆ STATISTIC() [20/20]
| STATISTIC | ( | NumOfPGOSplit | , |
| "Number of critical edge splits." | ) |
◆ sumEdgeCount()
Definition at line 988 of file PGOInstrumentation.cpp.
References E(), llvm::Total, and uint64_t.
◆ verifyFuncBFI()
|
static |
Definition at line 1978 of file PGOInstrumentation.cpp.
References ColdCountThreshold, DEBUG_TYPE, llvm::OptimizationRemarkEmitter::emit(), F, llvm::BlockFrequencyInfo::getBlockProfileCount(), Msg, llvm::Remark, and uint64_t.
Referenced by annotateAllFunctions().
Variable Documentation
◆ ValueProfKindDescr
Definition at line 276 of file PGOInstrumentation.cpp.
Generated on for LLVM by
1.14.0