LLVM  4.0.0
FuzzerMerge.cpp
Go to the documentation of this file.
1 //===- FuzzerMerge.cpp - merging corpora ----------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 // Merging corpora.
10 //===----------------------------------------------------------------------===//
11 
12 #include "FuzzerInternal.h"
13 #include "FuzzerIO.h"
14 #include "FuzzerMerge.h"
15 #include "FuzzerTracePC.h"
16 #include "FuzzerUtil.h"
17 
18 #include <fstream>
19 #include <iterator>
20 #include <sstream>
21 
22 namespace fuzzer {
23 
24 bool Merger::Parse(const std::string &Str, bool ParseCoverage) {
25  std::istringstream SS(Str);
26  return Parse(SS, ParseCoverage);
27 }
28 
29 void Merger::ParseOrExit(std::istream &IS, bool ParseCoverage) {
30  if (!Parse(IS, ParseCoverage)) {
31  Printf("MERGE: failed to parse the control file (unexpected error)\n");
32  exit(1);
33  }
34 }
35 
36 // The control file example:
37 //
38 // 3 # The number of inputs
39 // 1 # The number of inputs in the first corpus, <= the previous number
40 // file0
41 // file1
42 // file2 # One file name per line.
43 // STARTED 0 123 # FileID, file size
44 // DONE 0 1 4 6 8 # FileID COV1 COV2 ...
45 // STARTED 1 456 # If DONE is missing, the input crashed while processing.
46 // STARTED 2 567
47 // DONE 2 8 9
48 bool Merger::Parse(std::istream &IS, bool ParseCoverage) {
49  LastFailure.clear();
50  std::string Line;
51 
52  // Parse NumFiles.
53  if (!std::getline(IS, Line, '\n')) return false;
54  std::istringstream L1(Line);
55  size_t NumFiles = 0;
56  L1 >> NumFiles;
57  if (NumFiles == 0 || NumFiles > 10000000) return false;
58 
59  // Parse NumFilesInFirstCorpus.
60  if (!std::getline(IS, Line, '\n')) return false;
61  std::istringstream L2(Line);
62  NumFilesInFirstCorpus = NumFiles + 1;
64  if (NumFilesInFirstCorpus > NumFiles) return false;
65 
66  // Parse file names.
67  Files.resize(NumFiles);
68  for (size_t i = 0; i < NumFiles; i++)
69  if (!std::getline(IS, Files[i].Name, '\n'))
70  return false;
71 
72  // Parse STARTED and DONE lines.
73  size_t ExpectedStartMarker = 0;
74  const size_t kInvalidStartMarker = -1;
75  size_t LastSeenStartMarker = kInvalidStartMarker;
76  while (std::getline(IS, Line, '\n')) {
77  std::istringstream ISS1(Line);
78  std::string Marker;
79  size_t N;
80  ISS1 >> Marker;
81  ISS1 >> N;
82  if (Marker == "STARTED") {
83  // STARTED FILE_ID FILE_SIZE
84  if (ExpectedStartMarker != N)
85  return false;
86  ISS1 >> Files[ExpectedStartMarker].Size;
87  LastSeenStartMarker = ExpectedStartMarker;
88  assert(ExpectedStartMarker < Files.size());
89  ExpectedStartMarker++;
90  } else if (Marker == "DONE") {
91  // DONE FILE_SIZE COV1 COV2 COV3 ...
92  size_t CurrentFileIdx = N;
93  if (CurrentFileIdx != LastSeenStartMarker)
94  return false;
95  LastSeenStartMarker = kInvalidStartMarker;
96  if (ParseCoverage) {
97  auto &V = Files[CurrentFileIdx].Features;
98  V.clear();
99  while (ISS1 >> std::hex >> N)
100  V.push_back(N);
101  std::sort(V.begin(), V.end());
102  }
103  } else {
104  return false;
105  }
106  }
107  if (LastSeenStartMarker != kInvalidStartMarker)
108  LastFailure = Files[LastSeenStartMarker].Name;
109 
110  FirstNotProcessedFile = ExpectedStartMarker;
111  return true;
112 }
113 
114 // Decides which files need to be merged (add thost to NewFiles).
115 // Returns the number of new features added.
116 size_t Merger::Merge(std::vector<std::string> *NewFiles) {
117  NewFiles->clear();
119  std::set<uint32_t> AllFeatures;
120 
121  // What features are in the initial corpus?
122  for (size_t i = 0; i < NumFilesInFirstCorpus; i++) {
123  auto &Cur = Files[i].Features;
124  AllFeatures.insert(Cur.begin(), Cur.end());
125  }
126  size_t InitialNumFeatures = AllFeatures.size();
127 
128  // Remove all features that we already know from all other inputs.
129  for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) {
130  auto &Cur = Files[i].Features;
131  std::vector<uint32_t> Tmp;
132  std::set_difference(Cur.begin(), Cur.end(), AllFeatures.begin(),
133  AllFeatures.end(), std::inserter(Tmp, Tmp.begin()));
134  Cur.swap(Tmp);
135  }
136 
137  // Sort. Give preference to
138  // * smaller files
139  // * files with more features.
140  std::sort(Files.begin() + NumFilesInFirstCorpus, Files.end(),
141  [&](const MergeFileInfo &a, const MergeFileInfo &b) -> bool {
142  if (a.Size != b.Size)
143  return a.Size < b.Size;
144  return a.Features.size() > b.Features.size();
145  });
146 
147  // One greedy pass: add the file's features to AllFeatures.
148  // If new features were added, add this file to NewFiles.
149  for (size_t i = NumFilesInFirstCorpus; i < Files.size(); i++) {
150  auto &Cur = Files[i].Features;
151  // Printf("%s -> sz %zd ft %zd\n", Files[i].Name.c_str(),
152  // Files[i].Size, Cur.size());
153  size_t OldSize = AllFeatures.size();
154  AllFeatures.insert(Cur.begin(), Cur.end());
155  if (AllFeatures.size() > OldSize)
156  NewFiles->push_back(Files[i].Name);
157  }
158  return AllFeatures.size() - InitialNumFeatures;
159 }
160 
161 // Inner process. May crash if the target crashes.
162 void Fuzzer::CrashResistantMergeInternalStep(const std::string &CFPath) {
163  Printf("MERGE-INNER: using the control file '%s'\n", CFPath.c_str());
164  Merger M;
165  std::ifstream IF(CFPath);
166  M.ParseOrExit(IF, false);
167  IF.close();
168  if (!M.LastFailure.empty())
169  Printf("MERGE-INNER: '%s' caused a failure at the previous merge step\n",
170  M.LastFailure.c_str());
171 
172  Printf("MERGE-INNER: %zd total files;"
173  " %zd processed earlier; will process %zd files now\n",
174  M.Files.size(), M.FirstNotProcessedFile,
175  M.Files.size() - M.FirstNotProcessedFile);
176 
177  std::ofstream OF(CFPath, std::ofstream::out | std::ofstream::app);
178  for (size_t i = M.FirstNotProcessedFile; i < M.Files.size(); i++) {
179  auto U = FileToVector(M.Files[i].Name);
180  if (U.size() > MaxInputLen) {
181  U.resize(MaxInputLen);
182  U.shrink_to_fit();
183  }
184  std::ostringstream StartedLine;
185  // Write the pre-run marker.
186  OF << "STARTED " << std::dec << i << " " << U.size() << "\n";
187  OF.flush(); // Flush is important since ExecuteCommand may crash.
188  // Run.
189  TPC.ResetMaps();
190  ExecuteCallback(U.data(), U.size());
191  // Collect coverage.
192  std::set<size_t> Features;
193  TPC.CollectFeatures([&](size_t Feature) -> bool {
194  Features.insert(Feature);
195  return true;
196  });
197  // Show stats.
198  TotalNumberOfRuns++;
199  if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)))
200  PrintStats("pulse ");
201  // Write the post-run marker and the coverage.
202  OF << "DONE " << i;
203  for (size_t F : Features)
204  OF << " " << std::hex << F;
205  OF << "\n";
206  }
207 }
208 
209 // Outer process. Does not call the target code and thus sohuld not fail.
210 void Fuzzer::CrashResistantMerge(const std::vector<std::string> &Args,
211  const std::vector<std::string> &Corpora) {
212  if (Corpora.size() <= 1) {
213  Printf("Merge requires two or more corpus dirs\n");
214  return;
215  }
216  std::vector<std::string> AllFiles;
217  ListFilesInDirRecursive(Corpora[0], nullptr, &AllFiles, /*TopDir*/true);
218  size_t NumFilesInFirstCorpus = AllFiles.size();
219  for (size_t i = 1; i < Corpora.size(); i++)
220  ListFilesInDirRecursive(Corpora[i], nullptr, &AllFiles, /*TopDir*/true);
221  Printf("MERGE-OUTER: %zd files, %zd in the initial corpus\n",
222  AllFiles.size(), NumFilesInFirstCorpus);
223  auto CFPath = DirPlusFile(TmpDir(),
224  "libFuzzerTemp." + std::to_string(GetPid()) + ".txt");
225  // Write the control file.
226  RemoveFile(CFPath);
227  std::ofstream ControlFile(CFPath);
228  ControlFile << AllFiles.size() << "\n";
229  ControlFile << NumFilesInFirstCorpus << "\n";
230  for (auto &Path: AllFiles)
231  ControlFile << Path << "\n";
232  if (!ControlFile) {
233  Printf("MERGE-OUTER: failed to write to the control file: %s\n",
234  CFPath.c_str());
235  exit(1);
236  }
237  ControlFile.close();
238 
239  // Execute the inner process untill it passes.
240  // Every inner process should execute at least one input.
241  std::string BaseCmd = CloneArgsWithoutX(Args, "keep-all-flags");
242  for (size_t i = 1; i <= AllFiles.size(); i++) {
243  Printf("MERGE-OUTER: attempt %zd\n", i);
244  auto ExitCode =
245  ExecuteCommand(BaseCmd + " -merge_control_file=" + CFPath);
246  if (!ExitCode) {
247  Printf("MERGE-OUTER: succesfull in %zd attempt(s)\n", i);
248  break;
249  }
250  }
251  // Read the control file and do the merge.
252  Merger M;
253  std::ifstream IF(CFPath);
254  IF.seekg(0, IF.end);
255  Printf("MERGE-OUTER: the control file has %zd bytes\n", (size_t)IF.tellg());
256  IF.seekg(0, IF.beg);
257  M.ParseOrExit(IF, true);
258  IF.close();
259  std::vector<std::string> NewFiles;
260  size_t NumNewFeatures = M.Merge(&NewFiles);
261  Printf("MERGE-OUTER: %zd new files with %zd new features added\n",
262  NewFiles.size(), NumNewFeatures);
263  for (auto &F: NewFiles)
264  WriteToOutputCorpus(FileToVector(F));
265  // We are done, delete the control file.
266  RemoveFile(CFPath);
267 }
268 
269 } // namespace fuzzer
std::vector< uint32_t > Features
Definition: FuzzerMerge.h:53
size_t i
void CrashResistantMergeInternalStep(const std::string &ControlFilePath)
void ListFilesInDirRecursive(const std::string &Dir, long *Epoch, std::vector< std::string > *V, bool TopDir)
size_t Merge(std::vector< std::string > *NewFiles)
static Fuzzer * F
Definition: FuzzerLoop.cpp:59
size_t NumFilesInFirstCorpus
Definition: FuzzerMerge.h:58
bool Parse(std::istream &IS, bool ParseCoverage)
Definition: FuzzerMerge.cpp:48
void ParseOrExit(std::istream &IS, bool ParseCoverage)
Definition: FuzzerMerge.cpp:29
S1Ty set_difference(const S1Ty &S1, const S2Ty &S2)
set_difference(A, B) - Return A - B
Definition: SetOperations.h:51
TracePC TPC
void Printf(const char *Fmt,...)
Definition: FuzzerIO.cpp:109
void ExecuteCallback(const uint8_t *Data, size_t Size)
Definition: FuzzerLoop.cpp:532
std::string DirPlusFile(const std::string &DirPath, const std::string &FileName)
Definition: FuzzerIO.cpp:87
std::string LastFailure
Definition: FuzzerMerge.h:60
size_t FirstNotProcessedFile
Definition: FuzzerMerge.h:59
Unit FileToVector(const std::string &Path, size_t MaxSize, bool ExitOnError)
Definition: FuzzerIO.cpp:33
void CrashResistantMerge(const std::vector< std::string > &Args, const std::vector< std::string > &Corpora)
unsigned long GetPid()
void RemoveFile(const std::string &Path)
std::string TmpDir()
#define N
const std::string to_string(const T &Value)
Definition: ScopedPrinter.h:62
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::vector< MergeFileInfo > Files
Definition: FuzzerMerge.h:57
const FeatureBitset Features
size_t CollectFeatures(Callback CB)
int ExecuteCommand(const std::string &Command)
std::string CloneArgsWithoutX(const std::vector< std::string > &Args, const char *X1, const char *X2)