57 #define DEBUG_TYPE "insert-gcov-profiling"
73 cl::desc(
"Make counter updates atomic"));
78 return (
s.size() / 4) + 2;
101 GCOVProfiler() : GCOVProfiler(
GCOVOptions::getDefault()) {}
102 GCOVProfiler(
const GCOVOptions &Opts) : Options(Opts) {}
115 os->write(
s.data(),
s.size());
116 os->write_zeros(4 -
s.size() % 4);
118 void writeBytes(
const char *Bytes,
int Size) { os->write(Bytes,
Size); }
123 emitProfileNotes(
NamedMDNode *CUNode,
bool HasExecOrFork,
128 void emitGlobalConstructor(
131 bool isFunctionInstrumented(
const Function &
F);
132 std::vector<Regex> createRegexesFromString(
StringRef RegexesStr);
133 static bool doesFilenameMatchARegex(
StringRef Filename,
134 std::vector<Regex> &Regexes);
146 insertCounterWriteout(
ArrayRef<std::pair<GlobalVariable *, MDNode *>>);
149 bool AddFlushBeforeForkAndExec();
151 enum class GCovFileType { GCNO, GCDA };
165 std::vector<Regex> FilterRe;
166 std::vector<Regex> ExcludeRe;
171 class GCOVProfilerLegacyPass :
public ModulePass {
174 GCOVProfilerLegacyPass()
175 : GCOVProfilerLegacyPass(
GCOVOptions::getDefault()) {}
180 StringRef getPassName()
const override {
return "GCOV Profiler"; }
182 bool runOnModule(
Module &M)
override {
184 return &this->getAnalysis<BlockFrequencyInfoWrapperPass>(
F).getBFI();
187 return &this->getAnalysis<BranchProbabilityInfoWrapperPass>(
F).getBPI();
190 return this->getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
F);
192 return Profiler.runOnModule(M, GetBFI, GetBPI, GetTLI);
201 GCOVProfiler Profiler;
210 std::string infoString()
const {
224 bool Removed =
false;
225 bool IsCritical =
false;
228 : SrcBB(Src), DestBB(Dest), Weight(
W) {}
231 std::string infoString()
const {
232 return (
Twine(Removed ?
"-" :
" ") + (InMST ?
" " :
"*") +
233 (IsCritical ?
"c" :
" ") +
" W=" +
Twine(Weight))
241 GCOVProfilerLegacyPass,
"insert-gcov-profiling",
242 "Insert instrumentation for GCOV profiling",
false,
false)
251 return new GCOVProfilerLegacyPass(Options);
255 if (!SP->getLinkageName().empty())
256 return SP->getLinkageName();
280 GCOVRecord(GCOVProfiler *
P) :
P(
P) {}
284 void writeBytes(
const char *Bytes,
int Size) {
P->writeBytes(Bytes,
Size); }
293 class GCOVLines :
public GCOVRecord {
296 assert(Line != 0 &&
"Line zero is not a valid real line number.");
297 Lines.push_back(Line);
306 writeString(Filename);
307 for (
int i = 0,
e =
Lines.size();
i !=
e; ++
i)
315 std::string Filename;
326 return LinesByFile.try_emplace(Filename,
P, Filename).first->second;
330 OutEdges.emplace_back(&
Successor, Flags);
336 for (
auto &
I : LinesByFile) {
337 Len +=
I.second.length();
338 SortedLinesByFile.push_back(&
I);
349 for (
auto &
I : SortedLinesByFile)
350 I->getValue().writeOut();
359 assert(LinesByFile.empty());
382 : GCOVRecord(
P), SP(SP), EndLine(EndLine), Ident(Ident),
385 bool ExitBlockBeforeBody =
Version >= 48;
386 uint32_t i = ExitBlockBeforeBody ? 2 : 1;
389 if (!ExitBlockBeforeBody)
390 ReturnBlock.Number =
i;
392 std::string FunctionNameAndLine;
396 FuncChecksum =
hash_value(FunctionNameAndLine);
400 return Blocks.find(
const_cast<BasicBlock *
>(
BB))->second;
403 GCOVBlock &getEntryBlock() {
return EntryBlock; }
412 void writeOut(
uint32_t CfgChecksum) {
429 writeString(Filename);
430 write(SP->getLine());
432 write(SP->isArtificial());
433 writeString(Filename);
434 write(SP->getLine());
446 write(Blocks.size() + 2);
447 for (
int i = Blocks.size() + 2;
i; --
i)
451 write(Blocks.size() + 2);
456 const uint32_t Outgoing = EntryBlock.OutEdges.size();
459 write(Outgoing * 2 + 1);
460 write(EntryBlock.Number);
461 for (
const auto &
E : EntryBlock.OutEdges) {
466 for (
auto &It : Blocks) {
468 if (
Block.OutEdges.empty())
continue;
473 for (
const auto &
E :
Block.OutEdges) {
480 for (
auto &It : Blocks)
481 It.second.writeOut();
498 std::vector<Regex> GCOVProfiler::createRegexesFromString(
StringRef RegexesStr) {
499 std::vector<Regex> Regexes;
500 while (!RegexesStr.
empty()) {
501 std::pair<StringRef, StringRef> HeadTail = RegexesStr.
split(
';');
502 if (!HeadTail.first.empty()) {
503 Regex Re(HeadTail.first);
505 if (!Re.isValid(Err)) {
506 Ctx->emitError(
Twine(
"Regex ") + HeadTail.first +
507 " is not valid: " + Err);
511 RegexesStr = HeadTail.second;
516 bool GCOVProfiler::doesFilenameMatchARegex(
StringRef Filename,
517 std::vector<Regex> &Regexes) {
518 for (
Regex &Re : Regexes)
519 if (Re.match(Filename))
524 bool GCOVProfiler::isFunctionInstrumented(
const Function &
F) {
525 if (FilterRe.empty() && ExcludeRe.empty()) {
529 auto It = InstrumentedFiles.find(Filename);
530 if (It != InstrumentedFiles.end()) {
542 RealFilename = Filename;
544 RealFilename = RealPath;
547 bool ShouldInstrument;
548 if (FilterRe.empty()) {
549 ShouldInstrument = !doesFilenameMatchARegex(RealFilename, ExcludeRe);
550 }
else if (ExcludeRe.empty()) {
551 ShouldInstrument = doesFilenameMatchARegex(RealFilename, FilterRe);
553 ShouldInstrument = doesFilenameMatchARegex(RealFilename, FilterRe) &&
554 !doesFilenameMatchARegex(RealFilename, ExcludeRe);
556 InstrumentedFiles[Filename] = ShouldInstrument;
557 return ShouldInstrument;
561 GCovFileType OutputType) {
562 bool Notes = OutputType == GCovFileType::GCNO;
564 if (
NamedMDNode *GCov =
M->getNamedMetadata(
"llvm.gcov")) {
565 for (
int i = 0,
e = GCov->getNumOperands();
i !=
e; ++
i) {
567 bool ThreeElement =
N->getNumOperands() == 3;
568 if (!ThreeElement &&
N->getNumOperands() != 2)
570 if (dyn_cast<MDNode>(
N->getOperand(ThreeElement ? 2 : 1)) !=
CU)
576 MDString *NotesFile = dyn_cast<MDString>(
N->getOperand(0));
577 MDString *DataFile = dyn_cast<MDString>(
N->getOperand(1));
578 if (!NotesFile || !DataFile)
580 return std::string(Notes ? NotesFile->
getString()
584 MDString *GCovFile = dyn_cast<MDString>(
N->getOperand(0));
590 return std::string(Filename.
str());
599 return std::string(FName);
601 return std::string(CurPath.
str());
604 bool GCOVProfiler::runOnModule(
610 Ctx = &
M.getContext();
612 NamedMDNode *CUNode =
M.getNamedMetadata(
"llvm.dbg.cu");
613 if (!CUNode || (!Options.EmitNotes && !Options.EmitData))
616 bool HasExecOrFork = AddFlushBeforeForkAndExec();
618 FilterRe = createRegexesFromString(Options.Filter);
619 ExcludeRe = createRegexesFromString(Options.Exclude);
620 emitProfileNotes(CUNode, HasExecOrFork, GetBFI, GetBPI, this->GetTLI);
627 GCOVProfiler Profiler(GCOVOpts);
641 if (!Profiler.runOnModule(
M, GetBFI, GetBPI, GetTLI))
655 if (isa<DbgInfoIntrinsic>(&
I))
continue;
662 if (Loc.
getLine() == 0)
continue;
672 if (!
F.hasPersonalityFn())
return false;
678 bool GCOVProfiler::AddFlushBeforeForkAndExec() {
681 for (
auto &
F :
M->functions()) {
682 auto *TLI = &GetTLI(
F);
684 if (
CallInst *CI = dyn_cast<CallInst>(&
I)) {
685 if (
Function *Callee = CI->getCalledFunction()) {
687 if (TLI->getLibFunc(*Callee, LF)) {
688 if (LF == LibFunc_fork) {
692 }
else if (LF == LibFunc_execl || LF == LibFunc_execle ||
693 LF == LibFunc_execlp || LF == LibFunc_execv ||
694 LF == LibFunc_execvp || LF == LibFunc_execve ||
695 LF == LibFunc_execvpe || LF == LibFunc_execvP) {
704 for (
auto F : Forks) {
707 auto NextInst = ++
F->getIterator();
712 F->setCalledFunction(GCOVFork);
729 for (
auto E : Execs) {
732 auto NextInst = ++
E->getIterator();
738 M->getOrInsertFunction(
"llvm_writeout_files", FTy);
742 Builder.SetInsertPoint(&*NextInst);
745 FunctionCallee ResetF =
M->getOrInsertFunction(
"llvm_reset_counters", FTy);
746 Builder.CreateCall(ResetF)->setDebugLoc(Loc);
747 ExecBlocks.insert(Parent);
752 return !Forks.empty() || !Execs.empty();
757 if (
E.InMST ||
E.Removed)
763 if (SrcBB ==
nullptr)
765 if (DestBB ==
nullptr)
771 if (
BB->getFirstInsertionPt() ==
BB->end())
780 return CanInstrument(SrcBB);
782 return CanInstrument(DestBB);
792 MST.
addEdge(SrcBB, InstrBB, 0);
793 MST.
addEdge(InstrBB, DestBB, 0).InMST =
true;
796 return CanInstrument(InstrBB);
803 GCOVBlock &Src =
E.SrcBB ? GF.getBlock(
E.SrcBB) : GF.getEntryBlock();
804 GCOVBlock &Dst =
E.DestBB ? GF.getBlock(
E.DestBB) : GF.getReturnBlock();
805 dbgs() <<
" Edge " <<
ID++ <<
": " << Src.Number <<
"->" << Dst.Number
806 <<
E.infoString() <<
"\n";
811 bool GCOVProfiler::emitProfileNotes(
818 uint8_t c3 = Options.Version[0];
819 uint8_t
c2 = Options.Version[1];
820 uint8_t c1 = Options.Version[2];
821 Version = c3 >=
'A' ? (c3 -
'A') * 100 + (
c2 -
'0') * 10 + c1 -
'0'
822 : (c3 -
'0') * 10 + c1 -
'0';
825 bool EmitGCDA = Options.EmitData;
837 std::vector<uint8_t> EdgeDestinations;
842 unsigned FunctionIdent = 0;
843 for (
auto &
F :
M->functions()) {
867 for (
size_t I : llvm::seq<size_t>(0, MST.AllEdges.size())) {
868 auto &
E = *MST.AllEdges[
I];
877 Funcs.push_back(std::make_unique<GCOVFunction>(
this, &
F, SP, EndLine,
884 return E->Removed || (!E->InMST && !E->Place);
886 const size_t Measured =
887 std::stable_partition(
888 MST.AllEdges.begin(), MST.AllEdges.end(),
889 [](std::unique_ptr<Edge> &
E) { return E->Place; }) -
890 MST.AllEdges.begin();
891 for (
size_t I : llvm::seq<size_t>(0, Measured)) {
892 Edge &
E = *MST.AllEdges[
I];
894 E.SrcBB ?
Func.getBlock(
E.SrcBB) :
Func.getEntryBlock();
896 E.DestBB ?
Func.getBlock(
E.DestBB) :
Func.getReturnBlock();
897 E.SrcNumber = Src.Number;
898 E.DstNumber = Dst.Number;
901 MST.AllEdges.begin(), MST.AllEdges.begin() + Measured,
902 [](
const std::unique_ptr<Edge> &L,
const std::unique_ptr<Edge> &R) {
903 return L->SrcNumber != R->SrcNumber ? L->SrcNumber < R->SrcNumber
904 : L->DstNumber < R->DstNumber;
909 E.SrcBB ?
Func.getBlock(
E.SrcBB) :
Func.getEntryBlock();
911 E.DestBB ?
Func.getBlock(
E.DestBB) :
Func.getReturnBlock();
916 if (!SP->isArtificial())
917 Func.getBlock(&EntryBlock).getFile(Filename).addLine(Line);
921 for (
auto &GB :
Func.Blocks) {
923 auto &
Block = GB.second;
924 for (
auto Succ :
Block.OutEdges) {
926 do EdgeDestinations.push_back(Idx & 255);
927 while ((Idx >>= 8) > 0);
933 if (isa<DbgInfoIntrinsic>(&
I))
continue;
943 if (Line == Loc.
getLine())
continue;
961 for (
size_t I : llvm::seq<size_t>(0, Measured)) {
962 const Edge &
E = *MST.AllEdges[
I];
966 if (Options.Atomic) {
981 JC.
update(EdgeDestinations);
985 if (Options.EmitNotes) {
991 Twine(
"failed to open coverage notes file for writing: ") +
997 out.write(
"gcno", 4);
998 out.write(Options.Version, 4);
1000 out.write(
"oncg", 4);
1001 std::reverse_copy(Options.Version, Options.Version + 4, Tmp);
1010 for (
auto &Func : Funcs)
1011 Func->writeOut(Stamp);
1019 emitGlobalConstructor(CountersBySP);
1026 void GCOVProfiler::emitGlobalConstructor(
1027 SmallVectorImpl<std::pair<GlobalVariable *, MDNode *>> &CountersBySP) {
1028 Function *WriteoutF = insertCounterWriteout(CountersBySP);
1029 Function *ResetF = insertReset(CountersBySP);
1036 "__llvm_gcov_init", M);
1039 F->addFnAttr(Attribute::NoInline);
1040 if (Options.NoRedZone)
1041 F->addFnAttr(Attribute::NoRedZone);
1052 FunctionCallee GCOVInit =
M->getOrInsertFunction(
"llvm_gcov_init", FTy);
1053 Builder.CreateCall(GCOVInit, {WriteoutF, ResetF});
1068 AL =
AL.addParamAttribute(*Ctx, 2, AK);
1082 AL =
AL.addParamAttribute(*Ctx, 0, AK);
1083 AL =
AL.addParamAttribute(*Ctx, 1, AK);
1084 AL =
AL.addParamAttribute(*Ctx, 2, AK);
1086 return M->getOrInsertFunction(
"llvm_gcda_emit_function", FTy);
1097 AL =
AL.addParamAttribute(*Ctx, 0, AK);
1098 return M->getOrInsertFunction(
"llvm_gcda_emit_arcs", FTy,
AL);
1103 return M->getOrInsertFunction(
"llvm_gcda_summary_info", FTy);
1108 return M->getOrInsertFunction(
"llvm_gcda_end_file", FTy);
1111 Function *GCOVProfiler::insertCounterWriteout(
1112 ArrayRef<std::pair<GlobalVariable *, MDNode *> > CountersBySP) {
1114 Function *WriteoutF =
M->getFunction(
"__llvm_gcov_writeout");
1117 "__llvm_gcov_writeout", M);
1119 WriteoutF->
addFnAttr(Attribute::NoInline);
1120 if (Options.NoRedZone)
1121 WriteoutF->
addFnAttr(Attribute::NoRedZone);
1126 auto *TLI = &GetTLI(*WriteoutF);
1134 NamedMDNode *CUNodes =
M->getNamedMetadata(
"llvm.dbg.cu");
1144 "start_file_args_ty");
1147 "emit_function_args_ty");
1150 "emit_arcs_args_ty");
1153 EmitFunctionCallArgsTy->getPointerTo(),
1154 EmitArcsCallArgsTy->getPointerTo()},
1159 Constant *TwoZero32s[] = {Zero32, Zero32};
1169 std::string FilenameGcda = mangleName(
CU, GCovFileType::GCDA);
1172 StartFileCallArgsTy,
1173 {
Builder.CreateGlobalStringPtr(FilenameGcda),
1175 Builder.getInt32(CfgChecksum)});
1179 for (
int j : llvm::seq<int>(0, CountersBySP.size())) {
1180 uint32_t FuncChecksum = Funcs.empty() ? 0 : Funcs[
j]->getFuncChecksum();
1182 EmitFunctionCallArgsTy,
1184 Builder.getInt32(FuncChecksum),
1185 Builder.getInt32(CfgChecksum)}));
1188 unsigned Arcs = cast<ArrayType>(GV->
getValueType())->getNumElements();
1195 int CountersSize = CountersBySP.size();
1196 assert(CountersSize == (
int)EmitFunctionCallArgsArray.size() &&
1197 "Mismatched array size!");
1198 assert(CountersSize == (
int)EmitArcsCallArgsArray.size() &&
1199 "Mismatched array size!");
1200 auto *EmitFunctionCallArgsArrayTy =
1203 *M, EmitFunctionCallArgsArrayTy,
true,
1206 EmitFunctionCallArgsArray),
1207 Twine(
"__llvm_internal_gcov_emit_function_args.") +
Twine(
i));
1208 auto *EmitArcsCallArgsArrayTy =
1210 EmitFunctionCallArgsArrayGV->setUnnamedAddr(
1213 *M, EmitArcsCallArgsArrayTy,
true,
1216 Twine(
"__llvm_internal_gcov_emit_arcs_args.") +
Twine(
i));
1221 {StartFileCallArgs,
Builder.getInt32(CountersSize),
1223 EmitFunctionCallArgsArrayGV,
1226 EmitArcsCallArgsArrayTy, EmitArcsCallArgsArrayGV, TwoZero32s)}));
1230 if (FileInfos.empty()) {
1240 if ((int64_t)FileInfos.size() > (int64_t)INT_MAX)
1241 FileInfos.
resize(INT_MAX);
1245 auto *FileInfoArrayTy =
ArrayType::get(FileInfoTy, FileInfos.size());
1249 "__llvm_internal_gcov_emit_file_info");
1253 auto *FileLoopHeader =
1255 auto *CounterLoopHeader =
1261 Builder.CreateBr(FileLoopHeader);
1264 Builder.SetInsertPoint(FileLoopHeader);
1268 auto *FileInfoPtr =
Builder.CreateInBoundsGEP(
1269 FileInfoArrayTy, FileInfoArrayGV, {
Builder.getInt32(0), IV});
1270 auto *StartFileCallArgsPtr =
1271 Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 0,
"start_file_args");
1272 auto *StartFileCall =
Builder.CreateCall(
1275 Builder.CreateStructGEP(StartFileCallArgsTy,
1276 StartFileCallArgsPtr, 0),
1279 Builder.CreateStructGEP(StartFileCallArgsTy,
1280 StartFileCallArgsPtr, 1),
1283 Builder.CreateStructGEP(StartFileCallArgsTy,
1284 StartFileCallArgsPtr, 2),
1287 StartFileCall->addParamAttr(2, AK);
1288 auto *NumCounters =
Builder.CreateLoad(
1289 FileInfoTy->getElementType(1),
1290 Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 1),
"num_ctrs");
1291 auto *EmitFunctionCallArgsArray =
1292 Builder.CreateLoad(FileInfoTy->getElementType(2),
1293 Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 2),
1294 "emit_function_args");
1295 auto *EmitArcsCallArgsArray =
Builder.CreateLoad(
1296 FileInfoTy->getElementType(3),
1297 Builder.CreateStructGEP(FileInfoTy, FileInfoPtr, 3),
"emit_arcs_args");
1298 auto *EnterCounterLoopCond =
1300 Builder.CreateCondBr(EnterCounterLoopCond, CounterLoopHeader, FileLoopLatch);
1302 Builder.SetInsertPoint(CounterLoopHeader);
1305 JV->addIncoming(
Builder.getInt32(0), FileLoopHeader);
1306 auto *EmitFunctionCallArgsPtr =
Builder.CreateInBoundsGEP(
1307 EmitFunctionCallArgsTy, EmitFunctionCallArgsArray, JV);
1308 auto *EmitFunctionCall =
Builder.CreateCall(
1310 {
Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(0),
1311 Builder.CreateStructGEP(EmitFunctionCallArgsTy,
1312 EmitFunctionCallArgsPtr, 0),
1314 Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(1),
1315 Builder.CreateStructGEP(EmitFunctionCallArgsTy,
1316 EmitFunctionCallArgsPtr, 1),
1318 Builder.CreateLoad(EmitFunctionCallArgsTy->getElementType(2),
1319 Builder.CreateStructGEP(EmitFunctionCallArgsTy,
1320 EmitFunctionCallArgsPtr, 2),
1323 EmitFunctionCall->addParamAttr(0, AK);
1324 EmitFunctionCall->addParamAttr(1, AK);
1325 EmitFunctionCall->addParamAttr(2, AK);
1327 auto *EmitArcsCallArgsPtr =
1328 Builder.CreateInBoundsGEP(EmitArcsCallArgsTy, EmitArcsCallArgsArray, JV);
1329 auto *EmitArcsCall =
Builder.CreateCall(
1332 EmitArcsCallArgsTy->getElementType(0),
1333 Builder.CreateStructGEP(EmitArcsCallArgsTy, EmitArcsCallArgsPtr, 0),
1336 EmitArcsCallArgsTy->getElementType(1),
1337 Builder.CreateStructGEP(EmitArcsCallArgsTy, EmitArcsCallArgsPtr, 1),
1340 EmitArcsCall->addParamAttr(0, AK);
1342 auto *CounterLoopCond =
Builder.CreateICmpSLT(NextJV, NumCounters);
1343 Builder.CreateCondBr(CounterLoopCond, CounterLoopHeader, FileLoopLatch);
1344 JV->addIncoming(NextJV, CounterLoopHeader);
1346 Builder.SetInsertPoint(FileLoopLatch);
1347 Builder.CreateCall(SummaryInfo, {});
1348 Builder.CreateCall(EndFile, {});
1349 auto *NextIV =
Builder.CreateAdd(IV,
Builder.getInt32(1),
"next_file_idx");
1350 auto *FileLoopCond =
1351 Builder.CreateICmpSLT(NextIV,
Builder.getInt32(FileInfos.size()));
1352 Builder.CreateCondBr(FileLoopCond, FileLoopHeader, ExitBB);
1355 Builder.SetInsertPoint(ExitBB);
1361 Function *GCOVProfiler::insertReset(
1362 ArrayRef<std::pair<GlobalVariable *, MDNode *>> CountersBySP) {
1364 Function *ResetF =
M->getFunction(
"__llvm_gcov_reset");
1367 "__llvm_gcov_reset", M);
1370 if (Options.NoRedZone)
1371 ResetF->
addFnAttr(Attribute::NoRedZone);
1377 for (
const auto &
I : CountersBySP) {
1380 Builder.CreateStore(Null, GV);