File: | compiler-rt/lib/fuzzer/FuzzerDriver.cpp |
Warning: | line 759, column 12 Potential leak of memory pointed to by 'F' |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- FuzzerDriver.cpp - FuzzerDriver function and flags -----------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // FuzzerDriver and flag parsing. | |||
9 | //===----------------------------------------------------------------------===// | |||
10 | ||||
11 | #include "FuzzerCommand.h" | |||
12 | #include "FuzzerCorpus.h" | |||
13 | #include "FuzzerFork.h" | |||
14 | #include "FuzzerIO.h" | |||
15 | #include "FuzzerInterface.h" | |||
16 | #include "FuzzerInternal.h" | |||
17 | #include "FuzzerMerge.h" | |||
18 | #include "FuzzerMutate.h" | |||
19 | #include "FuzzerRandom.h" | |||
20 | #include "FuzzerTracePC.h" | |||
21 | #include <algorithm> | |||
22 | #include <atomic> | |||
23 | #include <chrono> | |||
24 | #include <cstdlib> | |||
25 | #include <cstring> | |||
26 | #include <mutex> | |||
27 | #include <string> | |||
28 | #include <thread> | |||
29 | #include <fstream> | |||
30 | ||||
31 | // This function should be present in the libFuzzer so that the client | |||
32 | // binary can test for its existence. | |||
33 | #if LIBFUZZER_MSVC0 | |||
34 | extern "C" void __libfuzzer_is_present() {} | |||
35 | #pragma comment(linker, "/include:__libfuzzer_is_present") | |||
36 | #else | |||
37 | extern "C" __attribute__((used)) void __libfuzzer_is_present() {} | |||
38 | #endif // LIBFUZZER_MSVC | |||
39 | ||||
40 | namespace fuzzer { | |||
41 | ||||
42 | // Program arguments. | |||
43 | struct FlagDescription { | |||
44 | const char *Name; | |||
45 | const char *Description; | |||
46 | int Default; | |||
47 | int *IntFlag; | |||
48 | const char **StrFlag; | |||
49 | unsigned int *UIntFlag; | |||
50 | }; | |||
51 | ||||
52 | struct { | |||
53 | #define FUZZER_DEPRECATED_FLAG(Name) | |||
54 | #define FUZZER_FLAG_INT(Name, Default, Description) int Name; | |||
55 | #define FUZZER_FLAG_UNSIGNED(Name, Default, Description) unsigned int Name; | |||
56 | #define FUZZER_FLAG_STRING(Name, Description) const char *Name; | |||
57 | #include "FuzzerFlags.def" | |||
58 | #undef FUZZER_DEPRECATED_FLAG | |||
59 | #undef FUZZER_FLAG_INT | |||
60 | #undef FUZZER_FLAG_UNSIGNED | |||
61 | #undef FUZZER_FLAG_STRING | |||
62 | } Flags; | |||
63 | ||||
64 | static const FlagDescription FlagDescriptions [] { | |||
65 | #define FUZZER_DEPRECATED_FLAG(Name) \ | |||
66 | {#Name, "Deprecated; don't use", 0, nullptr, nullptr, nullptr}, | |||
67 | #define FUZZER_FLAG_INT(Name, Default, Description) \ | |||
68 | {#Name, Description, Default, &Flags.Name, nullptr, nullptr}, | |||
69 | #define FUZZER_FLAG_UNSIGNED(Name, Default, Description) \ | |||
70 | {#Name, Description, static_cast<int>(Default), \ | |||
71 | nullptr, nullptr, &Flags.Name}, | |||
72 | #define FUZZER_FLAG_STRING(Name, Description) \ | |||
73 | {#Name, Description, 0, nullptr, &Flags.Name, nullptr}, | |||
74 | #include "FuzzerFlags.def" | |||
75 | #undef FUZZER_DEPRECATED_FLAG | |||
76 | #undef FUZZER_FLAG_INT | |||
77 | #undef FUZZER_FLAG_UNSIGNED | |||
78 | #undef FUZZER_FLAG_STRING | |||
79 | }; | |||
80 | ||||
81 | static const size_t kNumFlags = | |||
82 | sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]); | |||
83 | ||||
84 | static Vector<std::string> *Inputs; | |||
85 | static std::string *ProgName; | |||
86 | ||||
87 | static void PrintHelp() { | |||
88 | Printf("Usage:\n"); | |||
89 | auto Prog = ProgName->c_str(); | |||
90 | Printf("\nTo run fuzzing pass 0 or more directories.\n"); | |||
91 | Printf("%s [-flag1=val1 [-flag2=val2 ...] ] [dir1 [dir2 ...] ]\n", Prog); | |||
92 | ||||
93 | Printf("\nTo run individual tests without fuzzing pass 1 or more files:\n"); | |||
94 | Printf("%s [-flag1=val1 [-flag2=val2 ...] ] file1 [file2 ...]\n", Prog); | |||
95 | ||||
96 | Printf("\nFlags: (strictly in form -flag=value)\n"); | |||
97 | size_t MaxFlagLen = 0; | |||
98 | for (size_t F = 0; F < kNumFlags; F++) | |||
99 | MaxFlagLen = std::max(strlen(FlagDescriptions[F].Name), MaxFlagLen); | |||
100 | ||||
101 | for (size_t F = 0; F < kNumFlags; F++) { | |||
102 | const auto &D = FlagDescriptions[F]; | |||
103 | if (strstr(D.Description, "internal flag") == D.Description) continue; | |||
104 | Printf(" %s", D.Name); | |||
105 | for (size_t i = 0, n = MaxFlagLen - strlen(D.Name); i < n; i++) | |||
106 | Printf(" "); | |||
107 | Printf("\t"); | |||
108 | Printf("%d\t%s\n", D.Default, D.Description); | |||
109 | } | |||
110 | Printf("\nFlags starting with '--' will be ignored and " | |||
111 | "will be passed verbatim to subprocesses.\n"); | |||
112 | } | |||
113 | ||||
114 | static const char *FlagValue(const char *Param, const char *Name) { | |||
115 | size_t Len = strlen(Name); | |||
116 | if (Param[0] == '-' && strstr(Param + 1, Name) == Param + 1 && | |||
117 | Param[Len + 1] == '=') | |||
118 | return &Param[Len + 2]; | |||
119 | return nullptr; | |||
120 | } | |||
121 | ||||
122 | // Avoid calling stol as it triggers a bug in clang/glibc build. | |||
123 | static long MyStol(const char *Str) { | |||
124 | long Res = 0; | |||
125 | long Sign = 1; | |||
126 | if (*Str == '-') { | |||
127 | Str++; | |||
128 | Sign = -1; | |||
129 | } | |||
130 | for (size_t i = 0; Str[i]; i++) { | |||
131 | char Ch = Str[i]; | |||
132 | if (Ch < '0' || Ch > '9') | |||
133 | return Res; | |||
134 | Res = Res * 10 + (Ch - '0'); | |||
135 | } | |||
136 | return Res * Sign; | |||
137 | } | |||
138 | ||||
139 | static bool ParseOneFlag(const char *Param) { | |||
140 | if (Param[0] != '-') return false; | |||
141 | if (Param[1] == '-') { | |||
142 | static bool PrintedWarning = false; | |||
143 | if (!PrintedWarning) { | |||
144 | PrintedWarning = true; | |||
145 | Printf("INFO: libFuzzer ignores flags that start with '--'\n"); | |||
146 | } | |||
147 | for (size_t F = 0; F < kNumFlags; F++) | |||
148 | if (FlagValue(Param + 1, FlagDescriptions[F].Name)) | |||
149 | Printf("WARNING: did you mean '%s' (single dash)?\n", Param + 1); | |||
150 | return true; | |||
151 | } | |||
152 | for (size_t F = 0; F < kNumFlags; F++) { | |||
153 | const char *Name = FlagDescriptions[F].Name; | |||
154 | const char *Str = FlagValue(Param, Name); | |||
155 | if (Str) { | |||
156 | if (FlagDescriptions[F].IntFlag) { | |||
157 | int Val = MyStol(Str); | |||
158 | *FlagDescriptions[F].IntFlag = Val; | |||
159 | if (Flags.verbosity >= 2) | |||
160 | Printf("Flag: %s %d\n", Name, Val); | |||
161 | return true; | |||
162 | } else if (FlagDescriptions[F].UIntFlag) { | |||
163 | unsigned int Val = std::stoul(Str); | |||
164 | *FlagDescriptions[F].UIntFlag = Val; | |||
165 | if (Flags.verbosity >= 2) | |||
166 | Printf("Flag: %s %u\n", Name, Val); | |||
167 | return true; | |||
168 | } else if (FlagDescriptions[F].StrFlag) { | |||
169 | *FlagDescriptions[F].StrFlag = Str; | |||
170 | if (Flags.verbosity >= 2) | |||
171 | Printf("Flag: %s %s\n", Name, Str); | |||
172 | return true; | |||
173 | } else { // Deprecated flag. | |||
174 | Printf("Flag: %s: deprecated, don't use\n", Name); | |||
175 | return true; | |||
176 | } | |||
177 | } | |||
178 | } | |||
179 | Printf("\n\nWARNING: unrecognized flag '%s'; " | |||
180 | "use -help=1 to list all flags\n\n", Param); | |||
181 | return true; | |||
182 | } | |||
183 | ||||
184 | // We don't use any library to minimize dependencies. | |||
185 | static void ParseFlags(const Vector<std::string> &Args, | |||
186 | const ExternalFunctions *EF) { | |||
187 | for (size_t F = 0; F < kNumFlags; F++) { | |||
188 | if (FlagDescriptions[F].IntFlag) | |||
189 | *FlagDescriptions[F].IntFlag = FlagDescriptions[F].Default; | |||
190 | if (FlagDescriptions[F].UIntFlag) | |||
191 | *FlagDescriptions[F].UIntFlag = | |||
192 | static_cast<unsigned int>(FlagDescriptions[F].Default); | |||
193 | if (FlagDescriptions[F].StrFlag) | |||
194 | *FlagDescriptions[F].StrFlag = nullptr; | |||
195 | } | |||
196 | ||||
197 | // Disable len_control by default, if LLVMFuzzerCustomMutator is used. | |||
198 | if (EF->LLVMFuzzerCustomMutator) { | |||
199 | Flags.len_control = 0; | |||
200 | Printf("INFO: found LLVMFuzzerCustomMutator (%p). " | |||
201 | "Disabling -len_control by default.\n", EF->LLVMFuzzerCustomMutator); | |||
202 | } | |||
203 | ||||
204 | Inputs = new Vector<std::string>; | |||
205 | for (size_t A = 1; A < Args.size(); A++) { | |||
206 | if (ParseOneFlag(Args[A].c_str())) { | |||
207 | if (Flags.ignore_remaining_args) | |||
208 | break; | |||
209 | continue; | |||
210 | } | |||
211 | Inputs->push_back(Args[A]); | |||
212 | } | |||
213 | } | |||
214 | ||||
215 | static std::mutex Mu; | |||
216 | ||||
217 | static void PulseThread() { | |||
218 | while (true) { | |||
219 | SleepSeconds(600); | |||
220 | std::lock_guard<std::mutex> Lock(Mu); | |||
221 | Printf("pulse...\n"); | |||
222 | } | |||
223 | } | |||
224 | ||||
225 | static void WorkerThread(const Command &BaseCmd, std::atomic<unsigned> *Counter, | |||
226 | unsigned NumJobs, std::atomic<bool> *HasErrors) { | |||
227 | while (true) { | |||
228 | unsigned C = (*Counter)++; | |||
229 | if (C >= NumJobs) break; | |||
230 | std::string Log = "fuzz-" + std::to_string(C) + ".log"; | |||
231 | Command Cmd(BaseCmd); | |||
232 | Cmd.setOutputFile(Log); | |||
233 | Cmd.combineOutAndErr(); | |||
234 | if (Flags.verbosity) { | |||
235 | std::string CommandLine = Cmd.toString(); | |||
236 | Printf("%s\n", CommandLine.c_str()); | |||
237 | } | |||
238 | int ExitCode = ExecuteCommand(Cmd); | |||
239 | if (ExitCode != 0) | |||
240 | *HasErrors = true; | |||
241 | std::lock_guard<std::mutex> Lock(Mu); | |||
242 | Printf("================== Job %u exited with exit code %d ============\n", | |||
243 | C, ExitCode); | |||
244 | fuzzer::CopyFileToErr(Log); | |||
245 | } | |||
246 | } | |||
247 | ||||
248 | std::string CloneArgsWithoutX(const Vector<std::string> &Args, | |||
249 | const char *X1, const char *X2) { | |||
250 | std::string Cmd; | |||
251 | for (auto &S : Args) { | |||
252 | if (FlagValue(S.c_str(), X1) || FlagValue(S.c_str(), X2)) | |||
253 | continue; | |||
254 | Cmd += S + " "; | |||
255 | } | |||
256 | return Cmd; | |||
257 | } | |||
258 | ||||
259 | static int RunInMultipleProcesses(const Vector<std::string> &Args, | |||
260 | unsigned NumWorkers, unsigned NumJobs) { | |||
261 | std::atomic<unsigned> Counter(0); | |||
262 | std::atomic<bool> HasErrors(false); | |||
263 | Command Cmd(Args); | |||
264 | Cmd.removeFlag("jobs"); | |||
265 | Cmd.removeFlag("workers"); | |||
266 | Vector<std::thread> V; | |||
267 | std::thread Pulse(PulseThread); | |||
268 | Pulse.detach(); | |||
269 | for (unsigned i = 0; i < NumWorkers; i++) | |||
270 | V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors)); | |||
271 | for (auto &T : V) | |||
272 | T.join(); | |||
273 | return HasErrors ? 1 : 0; | |||
274 | } | |||
275 | ||||
276 | static void RssThread(Fuzzer *F, size_t RssLimitMb) { | |||
277 | while (true) { | |||
278 | SleepSeconds(1); | |||
279 | size_t Peak = GetPeakRSSMb(); | |||
280 | if (Peak > RssLimitMb) | |||
281 | F->RssLimitCallback(); | |||
282 | } | |||
283 | } | |||
284 | ||||
285 | static void StartRssThread(Fuzzer *F, size_t RssLimitMb) { | |||
286 | if (!RssLimitMb) | |||
287 | return; | |||
288 | std::thread T(RssThread, F, RssLimitMb); | |||
289 | T.detach(); | |||
290 | } | |||
291 | ||||
292 | int RunOneTest(Fuzzer *F, const char *InputFilePath, size_t MaxLen) { | |||
293 | Unit U = FileToVector(InputFilePath); | |||
294 | if (MaxLen && MaxLen < U.size()) | |||
295 | U.resize(MaxLen); | |||
296 | F->ExecuteCallback(U.data(), U.size()); | |||
297 | F->TryDetectingAMemoryLeak(U.data(), U.size(), true); | |||
298 | return 0; | |||
299 | } | |||
300 | ||||
301 | static bool AllInputsAreFiles() { | |||
302 | if (Inputs->empty()) return false; | |||
303 | for (auto &Path : *Inputs) | |||
304 | if (!IsFile(Path)) | |||
305 | return false; | |||
306 | return true; | |||
307 | } | |||
308 | ||||
309 | static std::string GetDedupTokenFromCmdOutput(const std::string &S) { | |||
310 | auto Beg = S.find("DEDUP_TOKEN:"); | |||
311 | if (Beg == std::string::npos) | |||
312 | return ""; | |||
313 | auto End = S.find('\n', Beg); | |||
314 | if (End == std::string::npos) | |||
315 | return ""; | |||
316 | return S.substr(Beg, End - Beg); | |||
317 | } | |||
318 | ||||
319 | int CleanseCrashInput(const Vector<std::string> &Args, | |||
320 | const FuzzingOptions &Options) { | |||
321 | if (Inputs->size() != 1 || !Flags.exact_artifact_path) { | |||
322 | Printf("ERROR: -cleanse_crash should be given one input file and" | |||
323 | " -exact_artifact_path\n"); | |||
324 | exit(1); | |||
325 | } | |||
326 | std::string InputFilePath = Inputs->at(0); | |||
327 | std::string OutputFilePath = Flags.exact_artifact_path; | |||
328 | Command Cmd(Args); | |||
329 | Cmd.removeFlag("cleanse_crash"); | |||
330 | ||||
331 | assert(Cmd.hasArgument(InputFilePath))((Cmd.hasArgument(InputFilePath)) ? static_cast<void> ( 0) : __assert_fail ("Cmd.hasArgument(InputFilePath)", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 331, __PRETTY_FUNCTION__)); | |||
332 | Cmd.removeArgument(InputFilePath); | |||
333 | ||||
334 | auto TmpFilePath = TempPath("CleanseCrashInput", ".repro"); | |||
335 | Cmd.addArgument(TmpFilePath); | |||
336 | Cmd.setOutputFile(getDevNull()); | |||
337 | Cmd.combineOutAndErr(); | |||
338 | ||||
339 | std::string CurrentFilePath = InputFilePath; | |||
340 | auto U = FileToVector(CurrentFilePath); | |||
341 | size_t Size = U.size(); | |||
342 | ||||
343 | const Vector<uint8_t> ReplacementBytes = {' ', 0xff}; | |||
344 | for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) { | |||
345 | bool Changed = false; | |||
346 | for (size_t Idx = 0; Idx < Size; Idx++) { | |||
347 | Printf("CLEANSE[%d]: Trying to replace byte %zd of %zd\n", NumAttempts, | |||
348 | Idx, Size); | |||
349 | uint8_t OriginalByte = U[Idx]; | |||
350 | if (ReplacementBytes.end() != std::find(ReplacementBytes.begin(), | |||
351 | ReplacementBytes.end(), | |||
352 | OriginalByte)) | |||
353 | continue; | |||
354 | for (auto NewByte : ReplacementBytes) { | |||
355 | U[Idx] = NewByte; | |||
356 | WriteToFile(U, TmpFilePath); | |||
357 | auto ExitCode = ExecuteCommand(Cmd); | |||
358 | RemoveFile(TmpFilePath); | |||
359 | if (!ExitCode) { | |||
360 | U[Idx] = OriginalByte; | |||
361 | } else { | |||
362 | Changed = true; | |||
363 | Printf("CLEANSE: Replaced byte %zd with 0x%x\n", Idx, NewByte); | |||
364 | WriteToFile(U, OutputFilePath); | |||
365 | break; | |||
366 | } | |||
367 | } | |||
368 | } | |||
369 | if (!Changed) break; | |||
370 | } | |||
371 | return 0; | |||
372 | } | |||
373 | ||||
374 | int MinimizeCrashInput(const Vector<std::string> &Args, | |||
375 | const FuzzingOptions &Options) { | |||
376 | if (Inputs->size() != 1) { | |||
377 | Printf("ERROR: -minimize_crash should be given one input file\n"); | |||
378 | exit(1); | |||
379 | } | |||
380 | std::string InputFilePath = Inputs->at(0); | |||
381 | Command BaseCmd(Args); | |||
382 | BaseCmd.removeFlag("minimize_crash"); | |||
383 | BaseCmd.removeFlag("exact_artifact_path"); | |||
384 | assert(BaseCmd.hasArgument(InputFilePath))((BaseCmd.hasArgument(InputFilePath)) ? static_cast<void> (0) : __assert_fail ("BaseCmd.hasArgument(InputFilePath)", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 384, __PRETTY_FUNCTION__)); | |||
385 | BaseCmd.removeArgument(InputFilePath); | |||
386 | if (Flags.runs <= 0 && Flags.max_total_time == 0) { | |||
387 | Printf("INFO: you need to specify -runs=N or " | |||
388 | "-max_total_time=N with -minimize_crash=1\n" | |||
389 | "INFO: defaulting to -max_total_time=600\n"); | |||
390 | BaseCmd.addFlag("max_total_time", "600"); | |||
391 | } | |||
392 | ||||
393 | BaseCmd.combineOutAndErr(); | |||
394 | ||||
395 | std::string CurrentFilePath = InputFilePath; | |||
396 | while (true) { | |||
397 | Unit U = FileToVector(CurrentFilePath); | |||
398 | Printf("CRASH_MIN: minimizing crash input: '%s' (%zd bytes)\n", | |||
399 | CurrentFilePath.c_str(), U.size()); | |||
400 | ||||
401 | Command Cmd(BaseCmd); | |||
402 | Cmd.addArgument(CurrentFilePath); | |||
403 | ||||
404 | Printf("CRASH_MIN: executing: %s\n", Cmd.toString().c_str()); | |||
405 | std::string CmdOutput; | |||
406 | bool Success = ExecuteCommand(Cmd, &CmdOutput); | |||
407 | if (Success) { | |||
408 | Printf("ERROR: the input %s did not crash\n", CurrentFilePath.c_str()); | |||
409 | exit(1); | |||
410 | } | |||
411 | Printf("CRASH_MIN: '%s' (%zd bytes) caused a crash. Will try to minimize " | |||
412 | "it further\n", | |||
413 | CurrentFilePath.c_str(), U.size()); | |||
414 | auto DedupToken1 = GetDedupTokenFromCmdOutput(CmdOutput); | |||
415 | if (!DedupToken1.empty()) | |||
416 | Printf("CRASH_MIN: DedupToken1: %s\n", DedupToken1.c_str()); | |||
417 | ||||
418 | std::string ArtifactPath = | |||
419 | Flags.exact_artifact_path | |||
420 | ? Flags.exact_artifact_path | |||
421 | : Options.ArtifactPrefix + "minimized-from-" + Hash(U); | |||
422 | Cmd.addFlag("minimize_crash_internal_step", "1"); | |||
423 | Cmd.addFlag("exact_artifact_path", ArtifactPath); | |||
424 | Printf("CRASH_MIN: executing: %s\n", Cmd.toString().c_str()); | |||
425 | CmdOutput.clear(); | |||
426 | Success = ExecuteCommand(Cmd, &CmdOutput); | |||
427 | Printf("%s", CmdOutput.c_str()); | |||
428 | if (Success) { | |||
429 | if (Flags.exact_artifact_path) { | |||
430 | CurrentFilePath = Flags.exact_artifact_path; | |||
431 | WriteToFile(U, CurrentFilePath); | |||
432 | } | |||
433 | Printf("CRASH_MIN: failed to minimize beyond %s (%d bytes), exiting\n", | |||
434 | CurrentFilePath.c_str(), U.size()); | |||
435 | break; | |||
436 | } | |||
437 | auto DedupToken2 = GetDedupTokenFromCmdOutput(CmdOutput); | |||
438 | if (!DedupToken2.empty()) | |||
439 | Printf("CRASH_MIN: DedupToken2: %s\n", DedupToken2.c_str()); | |||
440 | ||||
441 | if (DedupToken1 != DedupToken2) { | |||
442 | if (Flags.exact_artifact_path) { | |||
443 | CurrentFilePath = Flags.exact_artifact_path; | |||
444 | WriteToFile(U, CurrentFilePath); | |||
445 | } | |||
446 | Printf("CRASH_MIN: mismatch in dedup tokens" | |||
447 | " (looks like a different bug). Won't minimize further\n"); | |||
448 | break; | |||
449 | } | |||
450 | ||||
451 | CurrentFilePath = ArtifactPath; | |||
452 | Printf("*********************************\n"); | |||
453 | } | |||
454 | return 0; | |||
455 | } | |||
456 | ||||
457 | int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { | |||
458 | assert(Inputs->size() == 1)((Inputs->size() == 1) ? static_cast<void> (0) : __assert_fail ("Inputs->size() == 1", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 458, __PRETTY_FUNCTION__)); | |||
459 | std::string InputFilePath = Inputs->at(0); | |||
460 | Unit U = FileToVector(InputFilePath); | |||
461 | Printf("INFO: Starting MinimizeCrashInputInternalStep: %zd\n", U.size()); | |||
462 | if (U.size() < 2) { | |||
463 | Printf("INFO: The input is small enough, exiting\n"); | |||
464 | exit(0); | |||
465 | } | |||
466 | F->SetMaxInputLen(U.size()); | |||
467 | F->SetMaxMutationLen(U.size() - 1); | |||
468 | F->MinimizeCrashLoop(U); | |||
469 | Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n"); | |||
470 | exit(0); | |||
471 | return 0; | |||
472 | } | |||
473 | ||||
474 | void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, | |||
475 | const Vector<std::string> &Corpora, const char *CFPathOrNull) { | |||
476 | if (Corpora.size() < 2) { | |||
477 | Printf("INFO: Merge requires two or more corpus dirs\n"); | |||
478 | exit(0); | |||
479 | } | |||
480 | ||||
481 | Vector<SizedFile> OldCorpus, NewCorpus; | |||
482 | GetSizedFilesFromDir(Corpora[0], &OldCorpus); | |||
483 | for (size_t i = 1; i < Corpora.size(); i++) | |||
484 | GetSizedFilesFromDir(Corpora[i], &NewCorpus); | |||
485 | std::sort(OldCorpus.begin(), OldCorpus.end()); | |||
486 | std::sort(NewCorpus.begin(), NewCorpus.end()); | |||
487 | ||||
488 | std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath("Merge", ".txt"); | |||
489 | Vector<std::string> NewFiles; | |||
490 | Set<uint32_t> NewFeatures, NewCov; | |||
491 | CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, | |||
492 | {}, &NewCov, CFPath, true); | |||
493 | for (auto &Path : NewFiles) | |||
494 | F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); | |||
495 | // We are done, delete the control file if it was a temporary one. | |||
496 | if (!Flags.merge_control_file) | |||
497 | RemoveFile(CFPath); | |||
498 | ||||
499 | exit(0); | |||
500 | } | |||
501 | ||||
502 | int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, | |||
503 | UnitVector& Corpus) { | |||
504 | Printf("Started dictionary minimization (up to %d tests)\n", | |||
505 | Dict.size() * Corpus.size() * 2); | |||
506 | ||||
507 | // Scores and usage count for each dictionary unit. | |||
508 | Vector<int> Scores(Dict.size()); | |||
509 | Vector<int> Usages(Dict.size()); | |||
510 | ||||
511 | Vector<size_t> InitialFeatures; | |||
512 | Vector<size_t> ModifiedFeatures; | |||
513 | for (auto &C : Corpus) { | |||
514 | // Get coverage for the testcase without modifications. | |||
515 | F->ExecuteCallback(C.data(), C.size()); | |||
516 | InitialFeatures.clear(); | |||
517 | TPC.CollectFeatures([&](size_t Feature) { | |||
518 | InitialFeatures.push_back(Feature); | |||
519 | }); | |||
520 | ||||
521 | for (size_t i = 0; i < Dict.size(); ++i) { | |||
522 | Vector<uint8_t> Data = C; | |||
523 | auto StartPos = std::search(Data.begin(), Data.end(), | |||
524 | Dict[i].begin(), Dict[i].end()); | |||
525 | // Skip dictionary unit, if the testcase does not contain it. | |||
526 | if (StartPos == Data.end()) | |||
527 | continue; | |||
528 | ||||
529 | ++Usages[i]; | |||
530 | while (StartPos != Data.end()) { | |||
531 | // Replace all occurrences of dictionary unit in the testcase. | |||
532 | auto EndPos = StartPos + Dict[i].size(); | |||
533 | for (auto It = StartPos; It != EndPos; ++It) | |||
534 | *It ^= 0xFF; | |||
535 | ||||
536 | StartPos = std::search(EndPos, Data.end(), | |||
537 | Dict[i].begin(), Dict[i].end()); | |||
538 | } | |||
539 | ||||
540 | // Get coverage for testcase with masked occurrences of dictionary unit. | |||
541 | F->ExecuteCallback(Data.data(), Data.size()); | |||
542 | ModifiedFeatures.clear(); | |||
543 | TPC.CollectFeatures([&](size_t Feature) { | |||
544 | ModifiedFeatures.push_back(Feature); | |||
545 | }); | |||
546 | ||||
547 | if (InitialFeatures == ModifiedFeatures) | |||
548 | --Scores[i]; | |||
549 | else | |||
550 | Scores[i] += 2; | |||
551 | } | |||
552 | } | |||
553 | ||||
554 | Printf("###### Useless dictionary elements. ######\n"); | |||
555 | for (size_t i = 0; i < Dict.size(); ++i) { | |||
556 | // Dictionary units with positive score are treated as useful ones. | |||
557 | if (Scores[i] > 0) | |||
558 | continue; | |||
559 | ||||
560 | Printf("\""); | |||
561 | PrintASCII(Dict[i].data(), Dict[i].size(), "\""); | |||
562 | Printf(" # Score: %d, Used: %d\n", Scores[i], Usages[i]); | |||
563 | } | |||
564 | Printf("###### End of useless dictionary elements. ######\n"); | |||
565 | return 0; | |||
566 | } | |||
567 | ||||
568 | Vector<std::string> ParseSeedInuts(const char *seed_inputs) { | |||
569 | // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file | |||
570 | Vector<std::string> Files; | |||
571 | if (!seed_inputs) return Files; | |||
572 | std::string SeedInputs; | |||
573 | if (Flags.seed_inputs[0] == '@') | |||
574 | SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list. | |||
575 | else | |||
576 | SeedInputs = Flags.seed_inputs; // seed_inputs contains the list. | |||
577 | if (SeedInputs.empty()) { | |||
578 | Printf("seed_inputs is empty or @file does not exist.\n"); | |||
579 | exit(1); | |||
580 | } | |||
581 | // Parse SeedInputs. | |||
582 | size_t comma_pos = 0; | |||
583 | while ((comma_pos = SeedInputs.find_last_of(',')) != std::string::npos) { | |||
584 | Files.push_back(SeedInputs.substr(comma_pos + 1)); | |||
585 | SeedInputs = SeedInputs.substr(0, comma_pos); | |||
586 | } | |||
587 | Files.push_back(SeedInputs); | |||
588 | return Files; | |||
589 | } | |||
590 | ||||
591 | static Vector<SizedFile> ReadCorpora(const Vector<std::string> &CorpusDirs, | |||
592 | const Vector<std::string> &ExtraSeedFiles) { | |||
593 | Vector<SizedFile> SizedFiles; | |||
594 | size_t LastNumFiles = 0; | |||
595 | for (auto &Dir : CorpusDirs) { | |||
596 | GetSizedFilesFromDir(Dir, &SizedFiles); | |||
597 | Printf("INFO: % 8zd files found in %s\n", SizedFiles.size() - LastNumFiles, | |||
598 | Dir.c_str()); | |||
599 | LastNumFiles = SizedFiles.size(); | |||
600 | } | |||
601 | for (auto &File : ExtraSeedFiles) | |||
602 | if (auto Size = FileSize(File)) | |||
603 | SizedFiles.push_back({File, Size}); | |||
604 | return SizedFiles; | |||
605 | } | |||
606 | ||||
607 | int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { | |||
608 | using namespace fuzzer; | |||
609 | assert(argc && argv && "Argument pointers cannot be nullptr")((argc && argv && "Argument pointers cannot be nullptr" ) ? static_cast<void> (0) : __assert_fail ("argc && argv && \"Argument pointers cannot be nullptr\"" , "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 609, __PRETTY_FUNCTION__)); | |||
| ||||
610 | std::string Argv0((*argv)[0]); | |||
611 | EF = new ExternalFunctions(); | |||
612 | if (EF->LLVMFuzzerInitialize) | |||
613 | EF->LLVMFuzzerInitialize(argc, argv); | |||
614 | if (EF->__msan_scoped_disable_interceptor_checks) | |||
615 | EF->__msan_scoped_disable_interceptor_checks(); | |||
616 | const Vector<std::string> Args(*argv, *argv + *argc); | |||
617 | assert(!Args.empty())((!Args.empty()) ? static_cast<void> (0) : __assert_fail ("!Args.empty()", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 617, __PRETTY_FUNCTION__)); | |||
618 | ProgName = new std::string(Args[0]); | |||
619 | if (Argv0 != *ProgName) { | |||
620 | Printf("ERROR: argv[0] has been modified in LLVMFuzzerInitialize\n"); | |||
621 | exit(1); | |||
622 | } | |||
623 | ParseFlags(Args, EF); | |||
624 | if (Flags.help) { | |||
625 | PrintHelp(); | |||
626 | return 0; | |||
627 | } | |||
628 | ||||
629 | if (Flags.close_fd_mask & 2) | |||
630 | DupAndCloseStderr(); | |||
631 | if (Flags.close_fd_mask & 1) | |||
632 | CloseStdout(); | |||
633 | ||||
634 | if (Flags.jobs > 0 && Flags.workers == 0) { | |||
635 | Flags.workers = std::min(NumberOfCpuCores() / 2, Flags.jobs); | |||
636 | if (Flags.workers > 1) | |||
637 | Printf("Running %u workers\n", Flags.workers); | |||
638 | } | |||
639 | ||||
640 | if (Flags.workers > 0 && Flags.jobs > 0) | |||
641 | return RunInMultipleProcesses(Args, Flags.workers, Flags.jobs); | |||
642 | ||||
643 | FuzzingOptions Options; | |||
644 | Options.Verbosity = Flags.verbosity; | |||
645 | Options.MaxLen = Flags.max_len; | |||
646 | Options.LenControl = Flags.len_control; | |||
647 | Options.UnitTimeoutSec = Flags.timeout; | |||
648 | Options.ErrorExitCode = Flags.error_exitcode; | |||
649 | Options.TimeoutExitCode = Flags.timeout_exitcode; | |||
650 | Options.IgnoreTimeouts = Flags.ignore_timeouts; | |||
651 | Options.IgnoreOOMs = Flags.ignore_ooms; | |||
652 | Options.IgnoreCrashes = Flags.ignore_crashes; | |||
653 | Options.MaxTotalTimeSec = Flags.max_total_time; | |||
654 | Options.DoCrossOver = Flags.cross_over; | |||
655 | Options.MutateDepth = Flags.mutate_depth; | |||
656 | Options.ReduceDepth = Flags.reduce_depth; | |||
657 | Options.UseCounters = Flags.use_counters; | |||
658 | Options.UseMemmem = Flags.use_memmem; | |||
659 | Options.UseCmp = Flags.use_cmp; | |||
660 | Options.UseValueProfile = Flags.use_value_profile; | |||
661 | Options.Shrink = Flags.shrink; | |||
662 | Options.ReduceInputs = Flags.reduce_inputs; | |||
663 | Options.ShuffleAtStartUp = Flags.shuffle; | |||
664 | Options.PreferSmall = Flags.prefer_small; | |||
665 | Options.ReloadIntervalSec = Flags.reload; | |||
666 | Options.OnlyASCII = Flags.only_ascii; | |||
667 | Options.DetectLeaks = Flags.detect_leaks; | |||
668 | Options.PurgeAllocatorIntervalSec = Flags.purge_allocator_interval; | |||
669 | Options.TraceMalloc = Flags.trace_malloc; | |||
670 | Options.RssLimitMb = Flags.rss_limit_mb; | |||
671 | Options.MallocLimitMb = Flags.malloc_limit_mb; | |||
672 | if (!Options.MallocLimitMb) | |||
673 | Options.MallocLimitMb = Options.RssLimitMb; | |||
674 | if (Flags.runs >= 0) | |||
675 | Options.MaxNumberOfRuns = Flags.runs; | |||
676 | if (!Inputs->empty() && !Flags.minimize_crash_internal_step) | |||
677 | Options.OutputCorpus = (*Inputs)[0]; | |||
678 | Options.ReportSlowUnits = Flags.report_slow_units; | |||
679 | if (Flags.artifact_prefix) | |||
680 | Options.ArtifactPrefix = Flags.artifact_prefix; | |||
681 | if (Flags.exact_artifact_path) | |||
682 | Options.ExactArtifactPath = Flags.exact_artifact_path; | |||
683 | Vector<Unit> Dictionary; | |||
684 | if (Flags.dict) | |||
685 | if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary)) | |||
686 | return 1; | |||
687 | if (Flags.verbosity > 0 && !Dictionary.empty()) | |||
688 | Printf("Dictionary: %zd entries\n", Dictionary.size()); | |||
689 | bool RunIndividualFiles = AllInputsAreFiles(); | |||
690 | Options.SaveArtifacts = | |||
691 | !RunIndividualFiles
| |||
692 | Options.PrintNewCovPcs = Flags.print_pcs; | |||
693 | Options.PrintNewCovFuncs = Flags.print_funcs; | |||
694 | Options.PrintFinalStats = Flags.print_final_stats; | |||
695 | Options.PrintCorpusStats = Flags.print_corpus_stats; | |||
696 | Options.PrintCoverage = Flags.print_coverage; | |||
697 | if (Flags.exit_on_src_pos) | |||
698 | Options.ExitOnSrcPos = Flags.exit_on_src_pos; | |||
699 | if (Flags.exit_on_item) | |||
700 | Options.ExitOnItem = Flags.exit_on_item; | |||
701 | if (Flags.focus_function) | |||
702 | Options.FocusFunction = Flags.focus_function; | |||
703 | if (Flags.data_flow_trace) | |||
704 | Options.DataFlowTrace = Flags.data_flow_trace; | |||
705 | if (Flags.features_dir) | |||
706 | Options.FeaturesDir = Flags.features_dir; | |||
707 | if (Flags.collect_data_flow) | |||
708 | Options.CollectDataFlow = Flags.collect_data_flow; | |||
709 | if (Flags.stop_file) | |||
710 | Options.StopFile = Flags.stop_file; | |||
711 | ||||
712 | unsigned Seed = Flags.seed; | |||
713 | // Initialize Seed. | |||
714 | if (Seed == 0) | |||
715 | Seed = | |||
716 | std::chrono::system_clock::now().time_since_epoch().count() + GetPid(); | |||
717 | if (Flags.verbosity) | |||
718 | Printf("INFO: Seed: %u\n", Seed); | |||
719 | ||||
720 | if (Flags.collect_data_flow
| |||
721 | if (RunIndividualFiles) | |||
722 | return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, | |||
723 | ReadCorpora({}, *Inputs)); | |||
724 | else | |||
725 | return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, | |||
726 | ReadCorpora(*Inputs, {})); | |||
727 | } | |||
728 | ||||
729 | Random Rand(Seed); | |||
730 | auto *MD = new MutationDispatcher(Rand, Options); | |||
731 | auto *Corpus = new InputCorpus(Options.OutputCorpus); | |||
732 | auto *F = new Fuzzer(Callback, *Corpus, *MD, Options); | |||
733 | ||||
734 | for (auto &U: Dictionary) | |||
735 | if (U.size() <= Word::GetMaxSize()) | |||
736 | MD->AddWordToManualDictionary(Word(U.data(), U.size())); | |||
737 | ||||
738 | // Threads are only supported by Chrome. Don't use them with emscripten | |||
739 | // for now. | |||
740 | #if !LIBFUZZER_EMSCRIPTEN0 | |||
741 | StartRssThread(F, Flags.rss_limit_mb); | |||
742 | #endif // LIBFUZZER_EMSCRIPTEN | |||
743 | ||||
744 | Options.HandleAbrt = Flags.handle_abrt; | |||
745 | Options.HandleBus = Flags.handle_bus; | |||
746 | Options.HandleFpe = Flags.handle_fpe; | |||
747 | Options.HandleIll = Flags.handle_ill; | |||
748 | Options.HandleInt = Flags.handle_int; | |||
749 | Options.HandleSegv = Flags.handle_segv; | |||
750 | Options.HandleTerm = Flags.handle_term; | |||
751 | Options.HandleXfsz = Flags.handle_xfsz; | |||
752 | Options.HandleUsr1 = Flags.handle_usr1; | |||
753 | Options.HandleUsr2 = Flags.handle_usr2; | |||
754 | SetSignalHandler(Options); | |||
755 | ||||
756 | std::atexit(Fuzzer::StaticExitCallback); | |||
757 | ||||
758 | if (Flags.minimize_crash) | |||
759 | return MinimizeCrashInput(Args, Options); | |||
| ||||
760 | ||||
761 | if (Flags.minimize_crash_internal_step) | |||
762 | return MinimizeCrashInputInternalStep(F, Corpus); | |||
763 | ||||
764 | if (Flags.cleanse_crash) | |||
765 | return CleanseCrashInput(Args, Options); | |||
766 | ||||
767 | if (RunIndividualFiles) { | |||
768 | Options.SaveArtifacts = false; | |||
769 | int Runs = std::max(1, Flags.runs); | |||
770 | Printf("%s: Running %zd inputs %d time(s) each.\n", ProgName->c_str(), | |||
771 | Inputs->size(), Runs); | |||
772 | for (auto &Path : *Inputs) { | |||
773 | auto StartTime = system_clock::now(); | |||
774 | Printf("Running: %s\n", Path.c_str()); | |||
775 | for (int Iter = 0; Iter < Runs; Iter++) | |||
776 | RunOneTest(F, Path.c_str(), Options.MaxLen); | |||
777 | auto StopTime = system_clock::now(); | |||
778 | auto MS = duration_cast<milliseconds>(StopTime - StartTime).count(); | |||
779 | Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS); | |||
780 | } | |||
781 | Printf("***\n" | |||
782 | "*** NOTE: fuzzing was not performed, you have only\n" | |||
783 | "*** executed the target code on a fixed set of inputs.\n" | |||
784 | "***\n"); | |||
785 | F->PrintFinalStats(); | |||
786 | exit(0); | |||
787 | } | |||
788 | ||||
789 | if (Flags.fork) | |||
790 | FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); | |||
791 | ||||
792 | if (Flags.merge) | |||
793 | Merge(F, Options, Args, *Inputs, Flags.merge_control_file); | |||
794 | ||||
795 | if (Flags.merge_inner) { | |||
796 | const size_t kDefaultMaxMergeLen = 1 << 20; | |||
797 | if (Options.MaxLen == 0) | |||
798 | F->SetMaxInputLen(kDefaultMaxMergeLen); | |||
799 | assert(Flags.merge_control_file)((Flags.merge_control_file) ? static_cast<void> (0) : __assert_fail ("Flags.merge_control_file", "/build/llvm-toolchain-snapshot-11~++20200309111110+2c36c23f347/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 799, __PRETTY_FUNCTION__)); | |||
800 | F->CrashResistantMergeInternalStep(Flags.merge_control_file); | |||
801 | exit(0); | |||
802 | } | |||
803 | ||||
804 | if (Flags.analyze_dict) { | |||
805 | size_t MaxLen = INT_MAX2147483647; // Large max length. | |||
806 | UnitVector InitialCorpus; | |||
807 | for (auto &Inp : *Inputs) { | |||
808 | Printf("Loading corpus dir: %s\n", Inp.c_str()); | |||
809 | ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, | |||
810 | MaxLen, /*ExitOnError=*/false); | |||
811 | } | |||
812 | ||||
813 | if (Dictionary.empty() || Inputs->empty()) { | |||
814 | Printf("ERROR: can't analyze dict without dict and corpus provided\n"); | |||
815 | return 1; | |||
816 | } | |||
817 | if (AnalyzeDictionary(F, Dictionary, InitialCorpus)) { | |||
818 | Printf("Dictionary analysis failed\n"); | |||
819 | exit(1); | |||
820 | } | |||
821 | Printf("Dictionary analysis succeeded\n"); | |||
822 | exit(0); | |||
823 | } | |||
824 | ||||
825 | auto CorporaFiles = ReadCorpora(*Inputs, ParseSeedInuts(Flags.seed_inputs)); | |||
826 | F->Loop(CorporaFiles); | |||
827 | ||||
828 | if (Flags.verbosity) | |||
829 | Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(), | |||
830 | F->secondsSinceProcessStartUp()); | |||
831 | F->PrintFinalStats(); | |||
832 | ||||
833 | exit(0); // Don't let F destroy itself. | |||
834 | } | |||
835 | ||||
836 | // Storage for global ExternalFunctions object. | |||
837 | ExternalFunctions *EF = nullptr; | |||
838 | ||||
839 | } // namespace fuzzer |