37 cl::desc(
"Also inject (if missing) and verify MD_prof for "
38 "`select` instructions"));
41 cl::desc(
"Generate weights with small values for tests."));
44 "profcheck-default-select-true-weight",
cl::init(2U),
45 cl::desc(
"When annotating `select` instructions, this value will be used "
46 "for the first ('true') case."));
48 "profcheck-default-select-false-weight",
cl::init(3U),
49 cl::desc(
"When annotating `select` instructions, this value will be used "
50 "for the second ('false') case."));
52 "profcheck-annotate-indirect-calls",
cl::init(
false),
53 cl::desc(
"Also inject (if missing) and verify MD_prof for indirect calls"));
55 "profcheck-annotate-mem-ops",
cl::init(
false),
57 "Also inject (if missing) and verify MD_prof for memory operations"));
59class ProfileInjector {
65 getTerminatorBenefitingFromMDProf(
const BasicBlock &BB) {
75 static Instruction *getTerminatorBenefitingFromMDProf(BasicBlock &BB) {
77 getTerminatorBenefitingFromMDProf(
const_cast<const BasicBlock &
>(BB)));
85 if (!
F.hasFnAttribute(Attribute::AttrKind::Naked))
87 for (
const auto &BB :
F)
90 if (!CB || !CB->isInlineAsm())
97 F.getContext().emitError(
"Profile verification failed for function '" +
98 F.getName() +
"': " +
Msg);
105bool ProfileInjector::inject() {
127 if (!
F.getEntryCount())
131 if (*
F.getEntryCount() == 0)
137 uint32_t WeightsForTestOffset = 0;
139 for (Instruction &
I : BB) {
140 if (
I.getMetadata(LLVMContext::MD_prof))
144 if (
SI->getCondition()->getType()->isVectorTy())
156 InstrProfValueKind::IPVK_MemOPSize, 30);
162 if (!CB->isIndirectCall())
165 InstrProfValueKind::IPVK_IndirectCallTarget, 30);
170 auto *
Term = getTerminatorBenefitingFromMDProf(BB);
171 if (!Term ||
Term->getMetadata(LLVMContext::MD_prof))
175 SmallVector<uint32_t> Weights;
178 static const std::array Primes{3, 5, 7, 11, 13, 17, 19, 23, 29, 31,
179 37, 41, 43, 47, 53, 59, 61, 67, 71};
180 for (uint32_t
I = 0,
E =
Term->getNumSuccessors();
I <
E; ++
I)
182 Primes[(WeightsForTestOffset +
I) % Primes.size()]);
183 ++WeightsForTestOffset;
186 for (
auto I = 0U,
E =
Term->getNumSuccessors();
I <
E; ++
I)
190 [](
const BranchProbability &
P) {
191 return P.isUnknown();
193 "All branch probabilities should be valid");
194 const auto *FirstZeroDenominator =
195 find_if(Probs, [](
const BranchProbability &
P) {
196 return P.getDenominator() == 0;
198 (void)FirstZeroDenominator;
199 assert(FirstZeroDenominator == Probs.
end());
200 const auto *FirstNonZeroNumerator =
find_if(
201 Probs, [](
const BranchProbability &
P) {
return !
P.isZero(); });
202 assert(FirstNonZeroNumerator != Probs.
end());
203 DynamicAPInt LCM(Probs[0].getDenominator());
204 DynamicAPInt GCD(FirstNonZeroNumerator->getNumerator());
206 if (!Prob.getNumerator())
208 LCM =
llvm::lcm(LCM, DynamicAPInt(Prob.getDenominator()));
209 GCD =
llvm::gcd(GCD, DynamicAPInt(Prob.getNumerator()));
211 for (
const auto &Prob : Probs) {
213 (Prob.getNumerator() * LCM / GCD) / Prob.getDenominator();
214 Weights.
emplace_back(
static_cast<uint32_t
>((int64_t)W));
225 ProfileInjector PI(
F,
FAM);
234 auto PopulateIgnoreList = [&](
StringRef GVName) {
235 if (
const auto *CT = M.getGlobalVariable(GVName))
236 if (CT->hasInitializer())
239 for (
const auto &Elt : CA->operands())
241 if (CS->getNumOperands() >= 2 && CS->getOperand(1))
243 CS->getOperand(1)->stripPointerCasts()))
244 IgnoreList.insert(
F);
246 PopulateIgnoreList(
"llvm.global_ctors");
247 PopulateIgnoreList(
"llvm.global_dtors");
255 return PVP.run(
F,
FAM);
268 if (IgnoreList.contains(&
F))
271 const auto EntryCount =
F.getEntryCount();
273 auto *MD =
F.getMetadata(LLVMContext::MD_prof);
275 emitProfileError(
"function entry count missing (set to 0 if cold)",
F);
278 }
else if (*EntryCount == 0) {
281 for (
const auto &BB :
F) {
282 for (
const auto &
I : BB) {
283 if (
I.getMetadata(LLVMContext::MD_prof))
287 if (
SI->getCondition()->getType()->isVectorTy())
289 emitProfileError(
"select annotation missing",
F);
296 emitProfileError(
"memory operation annotation missing",
F);
301 if (!CB->isIndirectCall())
303 emitProfileError(
"indirect call annotation missing",
F);
307 if (
const auto *Term =
308 ProfileInjector::getTerminatorBenefitingFromMDProf(BB))
309 if (!
Term->getMetadata(LLVMContext::MD_prof))
310 emitProfileError(
"branch annotation missing",
F);
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
amdgpu aa AMDGPU Address space based Alias Analysis Wrapper
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Module.h This file contains the declarations for the Module class.
This header defines various interfaces for pass management in LLVM.
FunctionAnalysisManager FAM
ModuleAnalysisManager MAM
This file contains the declarations for profiling metadata utility functions.
static cl::opt< bool > WeightsForTest("profcheck-weights-for-test", cl::init(false), cl::desc("Generate weights with small values for tests."))
static cl::opt< uint32_t > SelectFalseWeight("profcheck-default-select-false-weight", cl::init(3U), cl::desc("When annotating `select` instructions, this value will be used " "for the second ('false') case."))
static cl::opt< int64_t > DefaultFunctionEntryCount("profcheck-default-function-entry-count", cl::init(1000))
static cl::opt< bool > AnnotateSelect("profcheck-annotate-select", cl::init(true), cl::desc("Also inject (if missing) and verify MD_prof for " "`select` instructions"))
static cl::opt< uint32_t > SelectTrueWeight("profcheck-default-select-true-weight", cl::init(2U), cl::desc("When annotating `select` instructions, this value will be used " "for the first ('true') case."))
static cl::opt< bool > AnnotateIndirectCalls("profcheck-annotate-indirect-calls", cl::init(false), cl::desc("Also inject (if missing) and verify MD_prof for indirect calls"))
static cl::opt< bool > AnnotateMemoryOperations("profcheck-annotate-mem-ops", cl::init(false), cl::desc("Also inject (if missing) and verify MD_prof for memory operations"))
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
LLVM Basic Block Representation.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction; assumes that the block is well-formed.
LLVM_ABI PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)
Runs the function pass across every function in the module.
A Module instance is used to store all the information related to an LLVM module.
A set of analyses that are preserved following a run of a transformation pass.
static PreservedAnalyses none()
Convenience factory function for the empty preserved set.
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM)
Checks that MD_prof is present on every instruction that supports it.
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
Represent a constant reference to a string, i.e.
Pass manager infrastructure for declaring and invalidating analyses.
@ BasicBlock
Various leaf nodes.
initializer< Ty > init(const Ty &Val)
friend class Instruction
Iterator for Instructions in a `BasicBlock.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt gcd(const DynamicAPInt &A, const DynamicAPInt &B)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
ModuleToFunctionPassAdaptor createModuleToFunctionPassAdaptor(FunctionPassT &&Pass, bool EagerlyInvalidate=false)
A function to deduce a function pass type and wrap it in the templated adaptor.
LLVM_ABI bool isExplicitlyUnknownProfileMetadata(const MDNode &MD)
auto dyn_cast_if_present(const Y &Val)
dyn_cast_if_present<X> - Functionally identical to dyn_cast, except that a null (or none in the case ...
LLVM_ABI void setBranchWeights(Instruction &I, ArrayRef< uint32_t > Weights, bool IsExpected, bool ElideAllZero=false)
Create a new branch_weights metadata node and add or overwrite a prof metadata reference to instructi...
LLVM_ABI void annotateValueSite(Module &M, Instruction &Inst, const InstrProfRecord &InstrProfR, InstrProfValueKind ValueKind, uint32_t SiteIndx, uint32_t MaxMDCount=3)
Get the value profile data for value site SiteIdx from InstrProfR and annotate the instruction Inst w...
auto succ_size(const MachineBasicBlock *BB)
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
auto drop_end(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the last N elements excluded.
LLVM_ATTRIBUTE_ALWAYS_INLINE DynamicAPInt lcm(const DynamicAPInt &A, const DynamicAPInt &B)
Returns the least common multiple of A and B.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
AnalysisManager< Function > FunctionAnalysisManager
Convenience typedef for the Function analysis manager.
AnalysisManager< Module > ModuleAnalysisManager
Convenience typedef for the Module analysis manager.
A CRTP mix-in for passes that can be skipped.