LLVM  4.0.0
FuzzerLoop.cpp
Go to the documentation of this file.
1 //===- FuzzerLoop.cpp - Fuzzer's main loop --------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 // Fuzzer's main loop.
10 //===----------------------------------------------------------------------===//
11 
12 #include "FuzzerCorpus.h"
13 #include "FuzzerInternal.h"
14 #include "FuzzerIO.h"
15 #include "FuzzerMutate.h"
16 #include "FuzzerRandom.h"
17 #include "FuzzerTracePC.h"
18 #include <algorithm>
19 #include <cstring>
20 #include <memory>
21 #include <set>
22 
23 #if defined(__has_include)
24 #if __has_include(<sanitizer / coverage_interface.h>)
25 #include <sanitizer/coverage_interface.h>
26 #endif
27 #if __has_include(<sanitizer / lsan_interface.h>)
28 #include <sanitizer/lsan_interface.h>
29 #endif
30 #endif
31 
32 #define NO_SANITIZE_MEMORY
33 #if defined(__has_feature)
34 #if __has_feature(memory_sanitizer)
35 #undef NO_SANITIZE_MEMORY
36 #define NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))
37 #endif
38 #endif
39 
40 namespace fuzzer {
41 static const size_t kMaxUnitSizeToPrint = 256;
42 
43 thread_local bool Fuzzer::IsMyThread;
44 
45 static void MissingExternalApiFunction(const char *FnName) {
46  Printf("ERROR: %s is not defined. Exiting.\n"
47  "Did you use -fsanitize-coverage=... to build your code?\n",
48  FnName);
49  exit(1);
50 }
51 
52 #define CHECK_EXTERNAL_FUNCTION(fn) \
53  do { \
54  if (!(EF->fn)) \
55  MissingExternalApiFunction(#fn); \
56  } while (false)
57 
58 // Only one Fuzzer per process.
59 static Fuzzer *F;
60 
61 void Fuzzer::ResetEdgeCoverage() {
62  CHECK_EXTERNAL_FUNCTION(__sanitizer_reset_coverage);
63  EF->__sanitizer_reset_coverage();
64 }
65 
66 void Fuzzer::ResetCounters() {
67  if (Options.UseCounters)
68  EF->__sanitizer_update_counter_bitset_and_clear_counters(0);
69 }
70 
71 void Fuzzer::PrepareCounters(Fuzzer::Coverage *C) {
72  if (Options.UseCounters) {
73  size_t NumCounters = EF->__sanitizer_get_number_of_counters();
74  C->CounterBitmap.resize(NumCounters);
75  }
76 }
77 
78 // Records data to a maximum coverage tracker. Returns true if additional
79 // coverage was discovered.
80 bool Fuzzer::RecordMaxCoverage(Fuzzer::Coverage *C) {
81  bool Res = false;
82 
83  uint64_t NewBlockCoverage = EF->__sanitizer_get_total_unique_coverage();
84  if (NewBlockCoverage > C->BlockCoverage) {
85  Res = true;
86  C->BlockCoverage = NewBlockCoverage;
87  }
88 
89  if (Options.UseIndirCalls &&
90  EF->__sanitizer_get_total_unique_caller_callee_pairs) {
91  uint64_t NewCallerCalleeCoverage =
92  EF->__sanitizer_get_total_unique_caller_callee_pairs();
93  if (NewCallerCalleeCoverage > C->CallerCalleeCoverage) {
94  Res = true;
95  C->CallerCalleeCoverage = NewCallerCalleeCoverage;
96  }
97  }
98 
99  if (Options.UseCounters) {
100  uint64_t CounterDelta =
101  EF->__sanitizer_update_counter_bitset_and_clear_counters(
102  C->CounterBitmap.data());
103  if (CounterDelta > 0) {
104  Res = true;
105  C->CounterBitmapBits += CounterDelta;
106  }
107  }
108 
109  return Res;
110 }
111 
112 // Leak detection is expensive, so we first check if there were more mallocs
113 // than frees (using the sanitizer malloc hooks) and only then try to call lsan.
115  void Start(int TraceLevel) {
116  this->TraceLevel = TraceLevel;
117  if (TraceLevel)
118  Printf("MallocFreeTracer: START\n");
119  Mallocs = 0;
120  Frees = 0;
121  }
122  // Returns true if there were more mallocs than frees.
123  bool Stop() {
124  if (TraceLevel)
125  Printf("MallocFreeTracer: STOP %zd %zd (%s)\n", Mallocs.load(),
126  Frees.load(), Mallocs == Frees ? "same" : "DIFFERENT");
127  bool Result = Mallocs > Frees;
128  Mallocs = 0;
129  Frees = 0;
130  TraceLevel = 0;
131  return Result;
132  }
133  std::atomic<size_t> Mallocs;
134  std::atomic<size_t> Frees;
135  int TraceLevel = 0;
136 };
137 
139 
141 void MallocHook(const volatile void *ptr, size_t size) {
142  size_t N = AllocTracer.Mallocs++;
143  F->HandleMalloc(size);
144  if (int TraceLevel = AllocTracer.TraceLevel) {
145  Printf("MALLOC[%zd] %p %zd\n", N, ptr, size);
146  if (TraceLevel >= 2 && EF)
147  EF->__sanitizer_print_stack_trace();
148  }
149 }
150 
152 void FreeHook(const volatile void *ptr) {
153  size_t N = AllocTracer.Frees++;
154  if (int TraceLevel = AllocTracer.TraceLevel) {
155  Printf("FREE[%zd] %p\n", N, ptr);
156  if (TraceLevel >= 2 && EF)
157  EF->__sanitizer_print_stack_trace();
158  }
159 }
160 
161 // Crash on a single malloc that exceeds the rss limit.
162 void Fuzzer::HandleMalloc(size_t Size) {
163  if (!Options.RssLimitMb || (Size >> 20) < (size_t)Options.RssLimitMb)
164  return;
165  Printf("==%d== ERROR: libFuzzer: out-of-memory (malloc(%zd))\n", GetPid(),
166  Size);
167  Printf(" To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
168  if (EF->__sanitizer_print_stack_trace)
169  EF->__sanitizer_print_stack_trace();
170  DumpCurrentUnit("oom-");
171  Printf("SUMMARY: libFuzzer: out-of-memory\n");
172  PrintFinalStats();
173  _Exit(Options.ErrorExitCode); // Stop right now.
174 }
175 
177  FuzzingOptions Options)
178  : CB(CB), Corpus(Corpus), MD(MD), Options(Options) {
179  SetDeathCallback();
181  assert(!F);
182  F = this;
183  TPC.ResetMaps();
184  ResetCoverage();
185  IsMyThread = true;
186  if (Options.DetectLeaks && EF->__sanitizer_install_malloc_and_free_hooks)
187  EF->__sanitizer_install_malloc_and_free_hooks(MallocHook, FreeHook);
188  TPC.SetUseCounters(Options.UseCounters);
191 
192  if (Options.Verbosity)
194  if (!Options.OutputCorpus.empty() && Options.ReloadIntervalSec)
195  EpochOfLastReadOfOutputCorpus = GetEpoch(Options.OutputCorpus);
196  MaxInputLen = MaxMutationLen = Options.MaxLen;
197  AllocateCurrentUnitData();
198  CurrentUnitSize = 0;
199  memset(BaseSha1, 0, sizeof(BaseSha1));
200 }
201 
203 
204 void Fuzzer::AllocateCurrentUnitData() {
205  if (CurrentUnitData || MaxInputLen == 0) return;
206  CurrentUnitData = new uint8_t[MaxInputLen];
207 }
208 
209 void Fuzzer::SetDeathCallback() {
210  CHECK_EXTERNAL_FUNCTION(__sanitizer_set_death_callback);
211  EF->__sanitizer_set_death_callback(StaticDeathCallback);
212 }
213 
214 void Fuzzer::StaticDeathCallback() {
215  assert(F);
216  F->DeathCallback();
217 }
218 
219 static void WarnOnUnsuccessfullMerge(bool DoWarn) {
220  if (!DoWarn) return;
221  Printf(
222  "***\n"
223  "***\n"
224  "***\n"
225  "*** NOTE: merge did not succeed due to a failure on one of the inputs.\n"
226  "*** You will need to filter out crashes from the corpus, e.g. like this:\n"
227  "*** for f in WITH_CRASHES/*; do ./fuzzer $f && cp $f NO_CRASHES; done\n"
228  "*** Future versions may have crash-resistant merge, stay tuned.\n"
229  "***\n"
230  "***\n"
231  "***\n");
232 }
233 
234 void Fuzzer::DumpCurrentUnit(const char *Prefix) {
235  WarnOnUnsuccessfullMerge(InMergeMode);
236  if (!CurrentUnitData) return; // Happens when running individual inputs.
238  Printf("; base unit: %s\n", Sha1ToString(BaseSha1).c_str());
239  size_t UnitSize = CurrentUnitSize;
240  if (UnitSize <= kMaxUnitSizeToPrint) {
241  PrintHexArray(CurrentUnitData, UnitSize, "\n");
242  PrintASCII(CurrentUnitData, UnitSize, "\n");
243  }
244  WriteUnitToFileWithPrefix({CurrentUnitData, CurrentUnitData + UnitSize},
245  Prefix);
246 }
247 
249 void Fuzzer::DeathCallback() {
250  DumpCurrentUnit("crash-");
251  PrintFinalStats();
252 }
253 
255  assert(F);
256  F->AlarmCallback();
257 }
258 
260  assert(F);
261  F->CrashCallback();
262 }
263 
265  assert(F);
266  F->InterruptCallback();
267 }
268 
270  Printf("==%lu== ERROR: libFuzzer: file size exceeded\n", GetPid());
271  exit(1);
272 }
273 
274 void Fuzzer::CrashCallback() {
275  Printf("==%lu== ERROR: libFuzzer: deadly signal\n", GetPid());
276  if (EF->__sanitizer_print_stack_trace)
277  EF->__sanitizer_print_stack_trace();
278  Printf("NOTE: libFuzzer has rudimentary signal handlers.\n"
279  " Combine libFuzzer with AddressSanitizer or similar for better "
280  "crash reports.\n");
281  Printf("SUMMARY: libFuzzer: deadly signal\n");
282  DumpCurrentUnit("crash-");
283  PrintFinalStats();
284  exit(Options.ErrorExitCode);
285 }
286 
287 void Fuzzer::InterruptCallback() {
288  Printf("==%lu== libFuzzer: run interrupted; exiting\n", GetPid());
289  PrintFinalStats();
290  _Exit(0); // Stop right now, don't perform any at-exit actions.
291 }
292 
294 void Fuzzer::AlarmCallback() {
295  assert(Options.UnitTimeoutSec > 0);
296  if (!InFuzzingThread()) return;
297  if (!RunningCB)
298  return; // We have not started running units yet.
299  size_t Seconds =
300  duration_cast<seconds>(system_clock::now() - UnitStartTime).count();
301  if (Seconds == 0)
302  return;
303  if (Options.Verbosity >= 2)
304  Printf("AlarmCallback %zd\n", Seconds);
305  if (Seconds >= (size_t)Options.UnitTimeoutSec) {
306  Printf("ALARM: working on the last Unit for %zd seconds\n", Seconds);
307  Printf(" and the timeout value is %d (use -timeout=N to change)\n",
308  Options.UnitTimeoutSec);
309  DumpCurrentUnit("timeout-");
310  Printf("==%lu== ERROR: libFuzzer: timeout after %d seconds\n", GetPid(),
311  Seconds);
312  if (EF->__sanitizer_print_stack_trace)
313  EF->__sanitizer_print_stack_trace();
314  Printf("SUMMARY: libFuzzer: timeout\n");
315  PrintFinalStats();
316  _Exit(Options.TimeoutExitCode); // Stop right now.
317  }
318 }
319 
321  Printf(
322  "==%lu== ERROR: libFuzzer: out-of-memory (used: %zdMb; limit: %zdMb)\n",
323  GetPid(), GetPeakRSSMb(), Options.RssLimitMb);
324  Printf(" To change the out-of-memory limit use -rss_limit_mb=<N>\n\n");
325  if (EF->__sanitizer_print_memory_profile)
326  EF->__sanitizer_print_memory_profile(95);
327  DumpCurrentUnit("oom-");
328  Printf("SUMMARY: libFuzzer: out-of-memory\n");
329  PrintFinalStats();
330  _Exit(Options.ErrorExitCode); // Stop right now.
331 }
332 
333 void Fuzzer::PrintStats(const char *Where, const char *End, size_t Units) {
334  size_t ExecPerSec = execPerSec();
335  if (Options.OutputCSV) {
336  static bool csvHeaderPrinted = false;
337  if (!csvHeaderPrinted) {
338  csvHeaderPrinted = true;
339  Printf("runs,block_cov,bits,cc_cov,corpus,execs_per_sec,tbms,reason\n");
340  }
341  Printf("%zd,%zd,%zd,%zd,%zd,%zd,%s\n", TotalNumberOfRuns,
342  MaxCoverage.BlockCoverage, MaxCoverage.CounterBitmapBits,
343  MaxCoverage.CallerCalleeCoverage, Corpus.size(), ExecPerSec, Where);
344  }
345 
346  if (!Options.Verbosity)
347  return;
348  Printf("#%zd\t%s", TotalNumberOfRuns, Where);
349  if (MaxCoverage.BlockCoverage)
350  Printf(" cov: %zd", MaxCoverage.BlockCoverage);
351  if (size_t N = MaxCoverage.VPMap.GetNumBitsSinceLastMerge())
352  Printf(" vp: %zd", N);
353  if (size_t N = TPC.GetTotalPCCoverage())
354  Printf(" cov: %zd", N);
355  if (auto TB = MaxCoverage.CounterBitmapBits)
356  Printf(" bits: %zd", TB);
357  if (size_t N = Corpus.NumFeatures())
358  Printf( " ft: %zd", N);
359  if (MaxCoverage.CallerCalleeCoverage)
360  Printf(" indir: %zd", MaxCoverage.CallerCalleeCoverage);
361  if (!Corpus.empty()) {
362  Printf(" corp: %zd", Corpus.NumActiveUnits());
363  if (size_t N = Corpus.SizeInBytes()) {
364  if (N < (1<<14))
365  Printf("/%zdb", N);
366  else if (N < (1 << 24))
367  Printf("/%zdKb", N >> 10);
368  else
369  Printf("/%zdMb", N >> 20);
370  }
371  }
372  if (Units)
373  Printf(" units: %zd", Units);
374 
375  Printf(" exec/s: %zd", ExecPerSec);
376  Printf(" rss: %zdMb", GetPeakRSSMb());
377  Printf("%s", End);
378 }
379 
381  if (Options.PrintCoverage)
382  TPC.PrintCoverage();
383  if (Options.DumpCoverage)
384  TPC.DumpCoverage();
385  if (Options.PrintCorpusStats)
386  Corpus.PrintStats();
387  if (!Options.PrintFinalStats) return;
388  size_t ExecPerSec = execPerSec();
389  Printf("stat::number_of_executed_units: %zd\n", TotalNumberOfRuns);
390  Printf("stat::average_exec_per_sec: %zd\n", ExecPerSec);
391  Printf("stat::new_units_added: %zd\n", NumberOfNewUnitsAdded);
392  Printf("stat::slowest_unit_time_sec: %zd\n", TimeOfLongestUnitInSeconds);
393  Printf("stat::peak_rss_mb: %zd\n", GetPeakRSSMb());
394 }
395 
396 void Fuzzer::SetMaxInputLen(size_t MaxInputLen) {
397  assert(this->MaxInputLen == 0); // Can only reset MaxInputLen from 0 to non-0.
398  assert(MaxInputLen);
399  this->MaxInputLen = MaxInputLen;
400  this->MaxMutationLen = MaxInputLen;
401  AllocateCurrentUnitData();
402  Printf("INFO: -max_len is not provided, using %zd\n", MaxInputLen);
403 }
404 
405 void Fuzzer::SetMaxMutationLen(size_t MaxMutationLen) {
406  assert(MaxMutationLen && MaxMutationLen <= MaxInputLen);
407  this->MaxMutationLen = MaxMutationLen;
408 }
409 
410 void Fuzzer::CheckExitOnSrcPosOrItem() {
411  if (!Options.ExitOnSrcPos.empty()) {
412  static auto *PCsSet = new std::set<uintptr_t>;
413  for (size_t i = 1, N = TPC.GetNumPCs(); i < N; i++) {
414  uintptr_t PC = TPC.GetPC(i);
415  if (!PC) continue;
416  if (!PCsSet->insert(PC).second) continue;
417  std::string Descr = DescribePC("%L", PC);
418  if (Descr.find(Options.ExitOnSrcPos) != std::string::npos) {
419  Printf("INFO: found line matching '%s', exiting.\n",
420  Options.ExitOnSrcPos.c_str());
421  _Exit(0);
422  }
423  }
424  }
425  if (!Options.ExitOnItem.empty()) {
426  if (Corpus.HasUnit(Options.ExitOnItem)) {
427  Printf("INFO: found item with checksum '%s', exiting.\n",
428  Options.ExitOnItem.c_str());
429  _Exit(0);
430  }
431  }
432 }
433 
434 void Fuzzer::RereadOutputCorpus(size_t MaxSize) {
435  if (Options.OutputCorpus.empty() || !Options.ReloadIntervalSec) return;
436  std::vector<Unit> AdditionalCorpus;
437  ReadDirToVectorOfUnits(Options.OutputCorpus.c_str(), &AdditionalCorpus,
438  &EpochOfLastReadOfOutputCorpus, MaxSize,
439  /*ExitOnError*/ false);
440  if (Options.Verbosity >= 2)
441  Printf("Reload: read %zd new units.\n", AdditionalCorpus.size());
442  bool Reloaded = false;
443  for (auto &U : AdditionalCorpus) {
444  if (U.size() > MaxSize)
445  U.resize(MaxSize);
446  if (!Corpus.HasUnit(U)) {
447  if (size_t NumFeatures = RunOne(U)) {
448  CheckExitOnSrcPosOrItem();
449  Corpus.AddToCorpus(U, NumFeatures);
450  Reloaded = true;
451  }
452  }
453  }
454  if (Reloaded)
455  PrintStats("RELOAD");
456 }
457 
458 void Fuzzer::ShuffleCorpus(UnitVector *V) {
459  std::random_shuffle(V->begin(), V->end(), MD.GetRand());
460  if (Options.PreferSmall)
461  std::stable_sort(V->begin(), V->end(), [](const Unit &A, const Unit &B) {
462  return A.size() < B.size();
463  });
464 }
465 
467  Printf("#0\tREAD units: %zd\n", InitialCorpus->size());
468  if (Options.ShuffleAtStartUp)
469  ShuffleCorpus(InitialCorpus);
470 
471  // Test the callback with empty input and never try it again.
472  uint8_t dummy;
473  ExecuteCallback(&dummy, 0);
474 
475  for (const auto &U : *InitialCorpus) {
476  if (size_t NumFeatures = RunOne(U)) {
477  CheckExitOnSrcPosOrItem();
478  Corpus.AddToCorpus(U, NumFeatures);
479  if (Options.Verbosity >= 2)
480  Printf("NEW0: %zd L %zd\n", MaxCoverage.BlockCoverage, U.size());
481  }
482  TryDetectingAMemoryLeak(U.data(), U.size(),
483  /*DuringInitialCorpusExecution*/ true);
484  }
485  PrintStats("INITED");
486  if (Corpus.empty()) {
487  Printf("ERROR: no interesting inputs were found. "
488  "Is the code instrumented for coverage? Exiting.\n");
489  exit(1);
490  }
491 }
492 
493 size_t Fuzzer::RunOne(const uint8_t *Data, size_t Size) {
494  if (!Size) return 0;
495  TotalNumberOfRuns++;
496 
497  ExecuteCallback(Data, Size);
498 
499  size_t Res = 0;
500  if (size_t NumFeatures = TPC.CollectFeatures([&](size_t Feature) -> bool {
501  return Corpus.AddFeature(Feature, Size, Options.Shrink);
502  }))
503  Res = NumFeatures;
504 
505  if (!TPC.UsingTracePcGuard()) {
506  if (TPC.UpdateValueProfileMap(&MaxCoverage.VPMap))
507  Res = 1;
508  if (!Res && RecordMaxCoverage(&MaxCoverage))
509  Res = 1;
510  }
511 
512  auto TimeOfUnit =
513  duration_cast<seconds>(UnitStopTime - UnitStartTime).count();
514  if (!(TotalNumberOfRuns & (TotalNumberOfRuns - 1)) &&
515  secondsSinceProcessStartUp() >= 2)
516  PrintStats("pulse ");
517  if (TimeOfUnit > TimeOfLongestUnitInSeconds * 1.1 &&
518  TimeOfUnit >= Options.ReportSlowUnits) {
519  TimeOfLongestUnitInSeconds = TimeOfUnit;
520  Printf("Slowest unit: %zd s:\n", TimeOfLongestUnitInSeconds);
521  WriteUnitToFileWithPrefix({Data, Data + Size}, "slow-unit-");
522  }
523  return Res;
524 }
525 
526 size_t Fuzzer::GetCurrentUnitInFuzzingThead(const uint8_t **Data) const {
527  assert(InFuzzingThread());
528  *Data = CurrentUnitData;
529  return CurrentUnitSize;
530 }
531 
532 void Fuzzer::ExecuteCallback(const uint8_t *Data, size_t Size) {
533  assert(InFuzzingThread());
534  // We copy the contents of Unit into a separate heap buffer
535  // so that we reliably find buffer overflows in it.
536  uint8_t *DataCopy = new uint8_t[Size];
537  memcpy(DataCopy, Data, Size);
538  if (CurrentUnitData && CurrentUnitData != Data)
539  memcpy(CurrentUnitData, Data, Size);
540  CurrentUnitSize = Size;
541  AllocTracer.Start(Options.TraceMalloc);
542  UnitStartTime = system_clock::now();
543  ResetCounters(); // Reset coverage right before the callback.
544  TPC.ResetMaps();
545  RunningCB = true;
546  int Res = CB(DataCopy, Size);
547  RunningCB = false;
548  UnitStopTime = system_clock::now();
549  (void)Res;
550  assert(Res == 0);
551  HasMoreMallocsThanFrees = AllocTracer.Stop();
552  CurrentUnitSize = 0;
553  delete[] DataCopy;
554 }
555 
556 void Fuzzer::WriteToOutputCorpus(const Unit &U) {
557  if (Options.OnlyASCII)
558  assert(IsASCII(U));
559  if (Options.OutputCorpus.empty())
560  return;
561  std::string Path = DirPlusFile(Options.OutputCorpus, Hash(U));
562  WriteToFile(U, Path);
563  if (Options.Verbosity >= 2)
564  Printf("Written to %s\n", Path.c_str());
565 }
566 
567 void Fuzzer::WriteUnitToFileWithPrefix(const Unit &U, const char *Prefix) {
568  if (!Options.SaveArtifacts)
569  return;
570  std::string Path = Options.ArtifactPrefix + Prefix + Hash(U);
571  if (!Options.ExactArtifactPath.empty())
572  Path = Options.ExactArtifactPath; // Overrides ArtifactPrefix.
573  WriteToFile(U, Path);
574  Printf("artifact_prefix='%s'; Test unit written to %s\n",
575  Options.ArtifactPrefix.c_str(), Path.c_str());
576  if (U.size() <= kMaxUnitSizeToPrint)
577  Printf("Base64: %s\n", Base64(U).c_str());
578 }
579 
580 void Fuzzer::PrintStatusForNewUnit(const Unit &U) {
581  if (!Options.PrintNEW)
582  return;
583  PrintStats("NEW ", "");
584  if (Options.Verbosity) {
585  Printf(" L: %zd ", U.size());
586  MD.PrintMutationSequence();
587  Printf("\n");
588  }
589 }
590 
591 void Fuzzer::ReportNewCoverage(InputInfo *II, const Unit &U) {
592  II->NumSuccessfullMutations++;
593  MD.RecordSuccessfulMutationSequence();
594  PrintStatusForNewUnit(U);
595  WriteToOutputCorpus(U);
596  NumberOfNewUnitsAdded++;
597  TPC.PrintNewPCs();
598 }
599 
600 // Finds minimal number of units in 'Extra' that add coverage to 'Initial'.
601 // We do it by actually executing the units, sometimes more than once,
602 // because we may be using different coverage-like signals and the only
603 // common thing between them is that we can say "this unit found new stuff".
604 UnitVector Fuzzer::FindExtraUnits(const UnitVector &Initial,
605  const UnitVector &Extra) {
606  UnitVector Res = Extra;
607  UnitVector Tmp;
608  size_t OldSize = Res.size();
609  for (int Iter = 0; Iter < 10; Iter++) {
610  ShuffleCorpus(&Res);
611  TPC.ResetMaps();
612  Corpus.ResetFeatureSet();
613  ResetCoverage();
614 
615  for (auto &U : Initial) {
616  TPC.ResetMaps();
617  RunOne(U);
618  }
619 
620  Tmp.clear();
621  for (auto &U : Res) {
622  TPC.ResetMaps();
623  if (RunOne(U))
624  Tmp.push_back(U);
625  }
626 
627  char Stat[7] = "MIN ";
628  Stat[3] = '0' + Iter;
629  PrintStats(Stat, "\n", Tmp.size());
630 
631  size_t NewSize = Tmp.size();
632  assert(NewSize <= OldSize);
633  Res.swap(Tmp);
634 
635  if (NewSize + 5 >= OldSize)
636  break;
637  OldSize = NewSize;
638  }
639  return Res;
640 }
641 
642 void Fuzzer::Merge(const std::vector<std::string> &Corpora) {
643  if (Corpora.size() <= 1) {
644  Printf("Merge requires two or more corpus dirs\n");
645  return;
646  }
647  InMergeMode = true;
648  std::vector<std::string> ExtraCorpora(Corpora.begin() + 1, Corpora.end());
649 
650  assert(MaxInputLen > 0);
651  UnitVector Initial, Extra;
652  ReadDirToVectorOfUnits(Corpora[0].c_str(), &Initial, nullptr, MaxInputLen,
653  true);
654  for (auto &C : ExtraCorpora)
655  ReadDirToVectorOfUnits(C.c_str(), &Extra, nullptr, MaxInputLen, true);
656 
657  if (!Initial.empty()) {
658  Printf("=== Minimizing the initial corpus of %zd units\n", Initial.size());
659  Initial = FindExtraUnits({}, Initial);
660  }
661 
662  Printf("=== Merging extra %zd units\n", Extra.size());
663  auto Res = FindExtraUnits(Initial, Extra);
664 
665  for (auto &U: Res)
666  WriteToOutputCorpus(U);
667 
668  Printf("=== Merge: written %zd units\n", Res.size());
669 }
670 
671 // Tries detecting a memory leak on the particular input that we have just
672 // executed before calling this function.
673 void Fuzzer::TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size,
674  bool DuringInitialCorpusExecution) {
675  if (!HasMoreMallocsThanFrees) return; // mallocs==frees, a leak is unlikely.
676  if (!Options.DetectLeaks) return;
677  if (!&(EF->__lsan_enable) || !&(EF->__lsan_disable) ||
678  !(EF->__lsan_do_recoverable_leak_check))
679  return; // No lsan.
680  // Run the target once again, but with lsan disabled so that if there is
681  // a real leak we do not report it twice.
682  EF->__lsan_disable();
683  ExecuteCallback(Data, Size);
684  EF->__lsan_enable();
685  if (!HasMoreMallocsThanFrees) return; // a leak is unlikely.
686  if (NumberOfLeakDetectionAttempts++ > 1000) {
687  Options.DetectLeaks = false;
688  Printf("INFO: libFuzzer disabled leak detection after every mutation.\n"
689  " Most likely the target function accumulates allocated\n"
690  " memory in a global state w/o actually leaking it.\n"
691  " You may try running this binary with -trace_malloc=[12]"
692  " to get a trace of mallocs and frees.\n"
693  " If LeakSanitizer is enabled in this process it will still\n"
694  " run on the process shutdown.\n");
695  return;
696  }
697  // Now perform the actual lsan pass. This is expensive and we must ensure
698  // we don't call it too often.
699  if (EF->__lsan_do_recoverable_leak_check()) { // Leak is found, report it.
700  if (DuringInitialCorpusExecution)
701  Printf("\nINFO: a leak has been found in the initial corpus.\n\n");
702  Printf("INFO: to ignore leaks on libFuzzer side use -detect_leaks=0.\n\n");
703  CurrentUnitSize = Size;
704  DumpCurrentUnit("leak-");
705  PrintFinalStats();
706  _Exit(Options.ErrorExitCode); // not exit() to disable lsan further on.
707  }
708 }
709 
710 static size_t ComputeMutationLen(size_t MaxInputSize, size_t MaxMutationLen,
711  Random &Rand) {
712  assert(MaxInputSize <= MaxMutationLen);
713  if (MaxInputSize == MaxMutationLen) return MaxMutationLen;
714  size_t Result = MaxInputSize;
715  size_t R = Rand.Rand();
716  if ((R % (1U << 7)) == 0)
717  Result++;
718  if ((R % (1U << 15)) == 0)
719  Result += 10 + Result / 2;
720  return Min(Result, MaxMutationLen);
721 }
722 
723 void Fuzzer::MutateAndTestOne() {
724  MD.StartMutationSequence();
725 
726  auto &II = Corpus.ChooseUnitToMutate(MD.GetRand());
727  const auto &U = II.U;
728  memcpy(BaseSha1, II.Sha1, sizeof(BaseSha1));
729  assert(CurrentUnitData);
730  size_t Size = U.size();
731  assert(Size <= MaxInputLen && "Oversized Unit");
732  memcpy(CurrentUnitData, U.data(), Size);
733 
734  assert(MaxMutationLen > 0);
735 
736  size_t CurrentMaxMutationLen =
737  Options.ExperimentalLenControl
738  ? ComputeMutationLen(Corpus.MaxInputSize(), MaxMutationLen,
739  MD.GetRand())
740  : MaxMutationLen;
741 
742  for (int i = 0; i < Options.MutateDepth; i++) {
743  if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
744  break;
745  size_t NewSize = 0;
746  NewSize = MD.Mutate(CurrentUnitData, Size, CurrentMaxMutationLen);
747  assert(NewSize > 0 && "Mutator returned empty unit");
748  assert(NewSize <= CurrentMaxMutationLen && "Mutator return overisized unit");
749  Size = NewSize;
750  if (i == 0)
751  StartTraceRecording();
752  II.NumExecutedMutations++;
753  if (size_t NumFeatures = RunOne(CurrentUnitData, Size)) {
754  Corpus.AddToCorpus({CurrentUnitData, CurrentUnitData + Size}, NumFeatures,
755  /*MayDeleteFile=*/true);
756  ReportNewCoverage(&II, {CurrentUnitData, CurrentUnitData + Size});
757  CheckExitOnSrcPosOrItem();
758  }
759  StopTraceRecording();
760  TryDetectingAMemoryLeak(CurrentUnitData, Size,
761  /*DuringInitialCorpusExecution*/ false);
762  }
763 }
764 
765 void Fuzzer::ResetCoverage() {
766  ResetEdgeCoverage();
767  MaxCoverage.Reset();
768  PrepareCounters(&MaxCoverage);
769 }
770 
771 void Fuzzer::Loop() {
773  system_clock::time_point LastCorpusReload = system_clock::now();
774  if (Options.DoCrossOver)
775  MD.SetCorpus(&Corpus);
776  while (true) {
777  auto Now = system_clock::now();
778  if (duration_cast<seconds>(Now - LastCorpusReload).count() >=
779  Options.ReloadIntervalSec) {
780  RereadOutputCorpus(MaxInputLen);
781  LastCorpusReload = system_clock::now();
782  }
783  if (TotalNumberOfRuns >= Options.MaxNumberOfRuns)
784  break;
785  if (TimedOut()) break;
786  // Perform several mutations and runs.
787  MutateAndTestOne();
788  }
789 
790  PrintStats("DONE ", "\n");
791  MD.PrintRecommendedDictionary();
792 }
793 
794 void Fuzzer::MinimizeCrashLoop(const Unit &U) {
795  if (U.size() <= 2) return;
796  while (!TimedOut() && TotalNumberOfRuns < Options.MaxNumberOfRuns) {
797  MD.StartMutationSequence();
798  memcpy(CurrentUnitData, U.data(), U.size());
799  for (int i = 0; i < Options.MutateDepth; i++) {
800  size_t NewSize = MD.Mutate(CurrentUnitData, U.size(), MaxMutationLen);
801  assert(NewSize > 0 && NewSize <= MaxMutationLen);
802  RunOne(CurrentUnitData, NewSize);
803  TryDetectingAMemoryLeak(CurrentUnitData, NewSize,
804  /*DuringInitialCorpusExecution*/ false);
805  }
806  }
807 }
808 
809 } // namespace fuzzer
810 
811 extern "C" {
812 
813 size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) {
814  assert(fuzzer::F);
815  return fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize);
816 }
817 } // extern "C"
void PrintHexArray(const uint8_t *Data, size_t Size, const char *PrintAfter)
Definition: FuzzerUtil.cpp:27
void SetPrintNewPCs(bool P)
Definition: FuzzerTracePC.h:57
int(* UserCallback)(const uint8_t *Data, size_t Size)
Definition: FuzzerDefs.h:73
size_t i
void SetMaxInputLen(size_t MaxInputLen)
Definition: FuzzerLoop.cpp:396
#define ATTRIBUTE_NO_SANITIZE_MEMORY
Definition: FuzzerDefs.h:51
bool InFuzzingThread() const
static void StaticFileSizeExceedCallback()
Definition: FuzzerLoop.cpp:269
static Fuzzer * F
Definition: FuzzerLoop.cpp:59
MutationDispatcher & GetMD()
ExternalFunctions * EF
size_t DefaultMutate(uint8_t *Data, size_t Size, size_t MaxSize)
Applies one of the default mutations.
static MallocFreeTracer AllocTracer
Definition: FuzzerLoop.cpp:138
uintptr_t GetPC(size_t Idx)
Definition: FuzzerTracePC.h:90
void ReadDirToVectorOfUnits(const char *Path, std::vector< Unit > *V, long *Epoch, size_t MaxSize, bool ExitOnError)
Definition: FuzzerIO.cpp:69
size_t GetNumBitsSinceLastMerge() const
void PrintMutationSequence()
Print the current sequence of mutations.
size_t execPerSec()
size_t RunOne(const uint8_t *Data, size_t Size)
Definition: FuzzerLoop.cpp:493
std::string DescribePC(const char *SymbolizedFMT, uintptr_t PC)
Definition: FuzzerUtil.cpp:182
long GetEpoch(const std::string &Path)
Definition: FuzzerIO.cpp:26
void ResetCoverage()
Definition: FuzzerLoop.cpp:765
static GCRegistry::Add< OcamlGC > B("ocaml","ocaml 3.10-compatible GC")
auto count(R &&Range, const E &Element) -> typename std::iterator_traits< decltype(std::begin(Range))>::difference_type
Wrapper function around std::count to count the number of times an element Element occurs in the give...
Definition: STLExtras.h:791
ATTRIBUTE_NO_SANITIZE_MEMORY void MallocHook(const volatile void *ptr, size_t size)
Definition: FuzzerLoop.cpp:141
TracePC TPC
void Printf(const char *Fmt,...)
Definition: FuzzerIO.cpp:109
T Min(T a, T b)
Definition: FuzzerDefs.h:56
std::string Base64(const Unit &U)
Definition: FuzzerUtil.cpp:154
Fuzzer(UserCallback CB, InputCorpus &Corpus, MutationDispatcher &MD, FuzzingOptions Options)
Definition: FuzzerLoop.cpp:176
void ExecuteCallback(const uint8_t *Data, size_t Size)
Definition: FuzzerLoop.cpp:532
size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize)
Definition: FuzzerLoop.cpp:813
std::string OutputCorpus
Definition: FuzzerOptions.h:43
static sys::TimePoint< std::chrono::seconds > now(bool Deterministic)
bool empty() const
Definition: FuzzerCorpus.h:69
std::string DirPlusFile(const std::string &DirPath, const std::string &FileName)
Definition: FuzzerIO.cpp:87
std::atomic< size_t > Frees
Definition: FuzzerLoop.cpp:134
bool UpdateValueProfileMap(ValueBitMap *MaxValueProfileMap)
Definition: FuzzerTracePC.h:59
static const unsigned End
static const size_t kMaxUnitSizeToPrint
Definition: FuzzerLoop.cpp:41
size_t GetTotalPCCoverage()
size_t size() const
Definition: FuzzerCorpus.h:50
void RssLimitCallback()
Definition: FuzzerLoop.cpp:320
void SetUseCounters(bool UC)
Definition: FuzzerTracePC.h:55
static size_t ComputeMutationLen(size_t MaxInputSize, size_t MaxMutationLen, Random &Rand)
Definition: FuzzerLoop.cpp:710
static void Merge(const std::string &Input, const std::vector< std::string > Result, size_t NumNewFeatures)
SmallVectorImpl< T >::const_pointer c_str(SmallVectorImpl< T > &str)
size_t NumActiveUnits() const
Definition: FuzzerCorpus.h:57
void WriteToFile(const Unit &U, const std::string &Path)
Definition: FuzzerIO.cpp:61
size_t Rand()
Definition: FuzzerRandom.h:21
void TryDetectingAMemoryLeak(const uint8_t *Data, size_t Size, bool DuringInitialCorpusExecution)
Definition: FuzzerLoop.cpp:673
static void MissingExternalApiFunction(const char *FnName)
Definition: FuzzerLoop.cpp:45
bool IsASCII(const Unit &U)
Definition: FuzzerUtil.cpp:73
size_t GetNumPCs() const
Definition: FuzzerTracePC.h:89
unsigned long GetPid()
static GCRegistry::Add< ShadowStackGC > C("shadow-stack","Very portable GC for uncooperative code generators")
size_t GetPeakRSSMb()
Definition: afl_driver.cpp:113
bool UsingTracePcGuard() const
Definition: FuzzerTracePC.h:81
void SetMaxMutationLen(size_t MaxMutationLen)
Definition: FuzzerLoop.cpp:405
void HandleMalloc(size_t Size)
Definition: FuzzerLoop.cpp:162
size_t SizeInBytes() const
Definition: FuzzerCorpus.h:51
void ShuffleAndMinimize(UnitVector *V)
Definition: FuzzerLoop.cpp:466
void InitializePrintNewPCs()
std::vector< Unit > UnitVector
Definition: FuzzerDefs.h:72
std::string ExitOnSrcPos
Definition: FuzzerOptions.h:46
std::string Sha1ToString(const uint8_t Sha1[kSHA1NumBytes])
Definition: FuzzerSHA1.cpp:209
void Start(int TraceLevel)
Definition: FuzzerLoop.cpp:115
#define N
static void StaticAlarmCallback()
Definition: FuzzerLoop.cpp:254
static void WarnOnUnsuccessfullMerge(bool DoWarn)
Definition: FuzzerLoop.cpp:219
std::atomic< size_t > Mallocs
Definition: FuzzerLoop.cpp:133
#define CHECK_EXTERNAL_FUNCTION(fn)
Definition: FuzzerLoop.cpp:52
std::vector< uint8_t > Unit
Definition: FuzzerDefs.h:71
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static void StaticInterruptCallback()
Definition: FuzzerLoop.cpp:264
size_t NumFeatures() const
Definition: FuzzerCorpus.h:158
bool HasUnit(const Unit &U)
Definition: FuzzerCorpus.h:88
std::string Hash(const Unit &U)
Definition: FuzzerSHA1.cpp:216
void PrintFinalStats()
Definition: FuzzerLoop.cpp:380
void SetUseValueProfile(bool VP)
Definition: FuzzerTracePC.h:56
bool AddFeature(size_t Idx, uint32_t NewSize, bool Shrink)
Definition: FuzzerCorpus.h:135
size_t CollectFeatures(Callback CB)
void RereadOutputCorpus(size_t MaxSize)
Definition: FuzzerLoop.cpp:434
#define NO_SANITIZE_MEMORY
Definition: FuzzerLoop.cpp:32
static GCRegistry::Add< ErlangGC > A("erlang","erlang-compatible garbage collector")
static void StaticCrashSignalCallback()
Definition: FuzzerLoop.cpp:259
ATTRIBUTE_NO_SANITIZE_MEMORY void FreeHook(const volatile void *ptr)
Definition: FuzzerLoop.cpp:152
static void PrintASCII(const Word &W, const char *PrintAfter)
char * PC
void AddToCorpus(const Unit &U, size_t NumFeatures, bool MayDeleteFile=false)
Definition: FuzzerCorpus.h:71