File: | projects/compiler-rt/lib/fuzzer/FuzzerDriver.cpp |
Warning: | line 757, column 12 Potential leak of memory pointed to by 'F' |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //===- FuzzerDriver.cpp - FuzzerDriver function and flags -----------------===// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | // FuzzerDriver and flag parsing. | |||
9 | //===----------------------------------------------------------------------===// | |||
10 | ||||
11 | #include "FuzzerCommand.h" | |||
12 | #include "FuzzerCorpus.h" | |||
13 | #include "FuzzerFork.h" | |||
14 | #include "FuzzerIO.h" | |||
15 | #include "FuzzerInterface.h" | |||
16 | #include "FuzzerInternal.h" | |||
17 | #include "FuzzerMerge.h" | |||
18 | #include "FuzzerMutate.h" | |||
19 | #include "FuzzerRandom.h" | |||
20 | #include "FuzzerTracePC.h" | |||
21 | #include <algorithm> | |||
22 | #include <atomic> | |||
23 | #include <chrono> | |||
24 | #include <cstdlib> | |||
25 | #include <cstring> | |||
26 | #include <mutex> | |||
27 | #include <string> | |||
28 | #include <thread> | |||
29 | #include <fstream> | |||
30 | ||||
31 | // This function should be present in the libFuzzer so that the client | |||
32 | // binary can test for its existence. | |||
33 | #if LIBFUZZER_MSVC0 | |||
34 | extern "C" void __libfuzzer_is_present() {} | |||
35 | #pragma comment(linker, "/include:__libfuzzer_is_present") | |||
36 | #else | |||
37 | extern "C" __attribute__((used)) void __libfuzzer_is_present() {} | |||
38 | #endif // LIBFUZZER_MSVC | |||
39 | ||||
40 | namespace fuzzer { | |||
41 | ||||
42 | // Program arguments. | |||
43 | struct FlagDescription { | |||
44 | const char *Name; | |||
45 | const char *Description; | |||
46 | int Default; | |||
47 | int *IntFlag; | |||
48 | const char **StrFlag; | |||
49 | unsigned int *UIntFlag; | |||
50 | }; | |||
51 | ||||
52 | struct { | |||
53 | #define FUZZER_DEPRECATED_FLAG(Name) | |||
54 | #define FUZZER_FLAG_INT(Name, Default, Description) int Name; | |||
55 | #define FUZZER_FLAG_UNSIGNED(Name, Default, Description) unsigned int Name; | |||
56 | #define FUZZER_FLAG_STRING(Name, Description) const char *Name; | |||
57 | #include "FuzzerFlags.def" | |||
58 | #undef FUZZER_DEPRECATED_FLAG | |||
59 | #undef FUZZER_FLAG_INT | |||
60 | #undef FUZZER_FLAG_UNSIGNED | |||
61 | #undef FUZZER_FLAG_STRING | |||
62 | } Flags; | |||
63 | ||||
64 | static const FlagDescription FlagDescriptions [] { | |||
65 | #define FUZZER_DEPRECATED_FLAG(Name) \ | |||
66 | {#Name, "Deprecated; don't use", 0, nullptr, nullptr, nullptr}, | |||
67 | #define FUZZER_FLAG_INT(Name, Default, Description) \ | |||
68 | {#Name, Description, Default, &Flags.Name, nullptr, nullptr}, | |||
69 | #define FUZZER_FLAG_UNSIGNED(Name, Default, Description) \ | |||
70 | {#Name, Description, static_cast<int>(Default), \ | |||
71 | nullptr, nullptr, &Flags.Name}, | |||
72 | #define FUZZER_FLAG_STRING(Name, Description) \ | |||
73 | {#Name, Description, 0, nullptr, &Flags.Name, nullptr}, | |||
74 | #include "FuzzerFlags.def" | |||
75 | #undef FUZZER_DEPRECATED_FLAG | |||
76 | #undef FUZZER_FLAG_INT | |||
77 | #undef FUZZER_FLAG_UNSIGNED | |||
78 | #undef FUZZER_FLAG_STRING | |||
79 | }; | |||
80 | ||||
81 | static const size_t kNumFlags = | |||
82 | sizeof(FlagDescriptions) / sizeof(FlagDescriptions[0]); | |||
83 | ||||
84 | static Vector<std::string> *Inputs; | |||
85 | static std::string *ProgName; | |||
86 | ||||
87 | static void PrintHelp() { | |||
88 | Printf("Usage:\n"); | |||
89 | auto Prog = ProgName->c_str(); | |||
90 | Printf("\nTo run fuzzing pass 0 or more directories.\n"); | |||
91 | Printf("%s [-flag1=val1 [-flag2=val2 ...] ] [dir1 [dir2 ...] ]\n", Prog); | |||
92 | ||||
93 | Printf("\nTo run individual tests without fuzzing pass 1 or more files:\n"); | |||
94 | Printf("%s [-flag1=val1 [-flag2=val2 ...] ] file1 [file2 ...]\n", Prog); | |||
95 | ||||
96 | Printf("\nFlags: (strictly in form -flag=value)\n"); | |||
97 | size_t MaxFlagLen = 0; | |||
98 | for (size_t F = 0; F < kNumFlags; F++) | |||
99 | MaxFlagLen = std::max(strlen(FlagDescriptions[F].Name), MaxFlagLen); | |||
100 | ||||
101 | for (size_t F = 0; F < kNumFlags; F++) { | |||
102 | const auto &D = FlagDescriptions[F]; | |||
103 | if (strstr(D.Description, "internal flag") == D.Description) continue; | |||
104 | Printf(" %s", D.Name); | |||
105 | for (size_t i = 0, n = MaxFlagLen - strlen(D.Name); i < n; i++) | |||
106 | Printf(" "); | |||
107 | Printf("\t"); | |||
108 | Printf("%d\t%s\n", D.Default, D.Description); | |||
109 | } | |||
110 | Printf("\nFlags starting with '--' will be ignored and " | |||
111 | "will be passed verbatim to subprocesses.\n"); | |||
112 | } | |||
113 | ||||
114 | static const char *FlagValue(const char *Param, const char *Name) { | |||
115 | size_t Len = strlen(Name); | |||
116 | if (Param[0] == '-' && strstr(Param + 1, Name) == Param + 1 && | |||
117 | Param[Len + 1] == '=') | |||
118 | return &Param[Len + 2]; | |||
119 | return nullptr; | |||
120 | } | |||
121 | ||||
122 | // Avoid calling stol as it triggers a bug in clang/glibc build. | |||
123 | static long MyStol(const char *Str) { | |||
124 | long Res = 0; | |||
125 | long Sign = 1; | |||
126 | if (*Str == '-') { | |||
127 | Str++; | |||
128 | Sign = -1; | |||
129 | } | |||
130 | for (size_t i = 0; Str[i]; i++) { | |||
131 | char Ch = Str[i]; | |||
132 | if (Ch < '0' || Ch > '9') | |||
133 | return Res; | |||
134 | Res = Res * 10 + (Ch - '0'); | |||
135 | } | |||
136 | return Res * Sign; | |||
137 | } | |||
138 | ||||
139 | static bool ParseOneFlag(const char *Param) { | |||
140 | if (Param[0] != '-') return false; | |||
141 | if (Param[1] == '-') { | |||
142 | static bool PrintedWarning = false; | |||
143 | if (!PrintedWarning) { | |||
144 | PrintedWarning = true; | |||
145 | Printf("INFO: libFuzzer ignores flags that start with '--'\n"); | |||
146 | } | |||
147 | for (size_t F = 0; F < kNumFlags; F++) | |||
148 | if (FlagValue(Param + 1, FlagDescriptions[F].Name)) | |||
149 | Printf("WARNING: did you mean '%s' (single dash)?\n", Param + 1); | |||
150 | return true; | |||
151 | } | |||
152 | for (size_t F = 0; F < kNumFlags; F++) { | |||
153 | const char *Name = FlagDescriptions[F].Name; | |||
154 | const char *Str = FlagValue(Param, Name); | |||
155 | if (Str) { | |||
156 | if (FlagDescriptions[F].IntFlag) { | |||
157 | int Val = MyStol(Str); | |||
158 | *FlagDescriptions[F].IntFlag = Val; | |||
159 | if (Flags.verbosity >= 2) | |||
160 | Printf("Flag: %s %d\n", Name, Val); | |||
161 | return true; | |||
162 | } else if (FlagDescriptions[F].UIntFlag) { | |||
163 | unsigned int Val = std::stoul(Str); | |||
164 | *FlagDescriptions[F].UIntFlag = Val; | |||
165 | if (Flags.verbosity >= 2) | |||
166 | Printf("Flag: %s %u\n", Name, Val); | |||
167 | return true; | |||
168 | } else if (FlagDescriptions[F].StrFlag) { | |||
169 | *FlagDescriptions[F].StrFlag = Str; | |||
170 | if (Flags.verbosity >= 2) | |||
171 | Printf("Flag: %s %s\n", Name, Str); | |||
172 | return true; | |||
173 | } else { // Deprecated flag. | |||
174 | Printf("Flag: %s: deprecated, don't use\n", Name); | |||
175 | return true; | |||
176 | } | |||
177 | } | |||
178 | } | |||
179 | Printf("\n\nWARNING: unrecognized flag '%s'; " | |||
180 | "use -help=1 to list all flags\n\n", Param); | |||
181 | return true; | |||
182 | } | |||
183 | ||||
184 | // We don't use any library to minimize dependencies. | |||
185 | static void ParseFlags(const Vector<std::string> &Args, | |||
186 | const ExternalFunctions *EF) { | |||
187 | for (size_t F = 0; F < kNumFlags; F++) { | |||
188 | if (FlagDescriptions[F].IntFlag) | |||
189 | *FlagDescriptions[F].IntFlag = FlagDescriptions[F].Default; | |||
190 | if (FlagDescriptions[F].UIntFlag) | |||
191 | *FlagDescriptions[F].UIntFlag = | |||
192 | static_cast<unsigned int>(FlagDescriptions[F].Default); | |||
193 | if (FlagDescriptions[F].StrFlag) | |||
194 | *FlagDescriptions[F].StrFlag = nullptr; | |||
195 | } | |||
196 | ||||
197 | // Disable len_control by default, if LLVMFuzzerCustomMutator is used. | |||
198 | if (EF->LLVMFuzzerCustomMutator) | |||
199 | Flags.len_control = 0; | |||
200 | ||||
201 | Inputs = new Vector<std::string>; | |||
202 | for (size_t A = 1; A < Args.size(); A++) { | |||
203 | if (ParseOneFlag(Args[A].c_str())) { | |||
204 | if (Flags.ignore_remaining_args) | |||
205 | break; | |||
206 | continue; | |||
207 | } | |||
208 | Inputs->push_back(Args[A]); | |||
209 | } | |||
210 | } | |||
211 | ||||
212 | static std::mutex Mu; | |||
213 | ||||
214 | static void PulseThread() { | |||
215 | while (true) { | |||
216 | SleepSeconds(600); | |||
217 | std::lock_guard<std::mutex> Lock(Mu); | |||
218 | Printf("pulse...\n"); | |||
219 | } | |||
220 | } | |||
221 | ||||
222 | static void WorkerThread(const Command &BaseCmd, std::atomic<unsigned> *Counter, | |||
223 | unsigned NumJobs, std::atomic<bool> *HasErrors) { | |||
224 | while (true) { | |||
225 | unsigned C = (*Counter)++; | |||
226 | if (C >= NumJobs) break; | |||
227 | std::string Log = "fuzz-" + std::to_string(C) + ".log"; | |||
228 | Command Cmd(BaseCmd); | |||
229 | Cmd.setOutputFile(Log); | |||
230 | Cmd.combineOutAndErr(); | |||
231 | if (Flags.verbosity) { | |||
232 | std::string CommandLine = Cmd.toString(); | |||
233 | Printf("%s\n", CommandLine.c_str()); | |||
234 | } | |||
235 | int ExitCode = ExecuteCommand(Cmd); | |||
236 | if (ExitCode != 0) | |||
237 | *HasErrors = true; | |||
238 | std::lock_guard<std::mutex> Lock(Mu); | |||
239 | Printf("================== Job %u exited with exit code %d ============\n", | |||
240 | C, ExitCode); | |||
241 | fuzzer::CopyFileToErr(Log); | |||
242 | } | |||
243 | } | |||
244 | ||||
245 | std::string CloneArgsWithoutX(const Vector<std::string> &Args, | |||
246 | const char *X1, const char *X2) { | |||
247 | std::string Cmd; | |||
248 | for (auto &S : Args) { | |||
249 | if (FlagValue(S.c_str(), X1) || FlagValue(S.c_str(), X2)) | |||
250 | continue; | |||
251 | Cmd += S + " "; | |||
252 | } | |||
253 | return Cmd; | |||
254 | } | |||
255 | ||||
256 | static int RunInMultipleProcesses(const Vector<std::string> &Args, | |||
257 | unsigned NumWorkers, unsigned NumJobs) { | |||
258 | std::atomic<unsigned> Counter(0); | |||
259 | std::atomic<bool> HasErrors(false); | |||
260 | Command Cmd(Args); | |||
261 | Cmd.removeFlag("jobs"); | |||
262 | Cmd.removeFlag("workers"); | |||
263 | Vector<std::thread> V; | |||
264 | std::thread Pulse(PulseThread); | |||
265 | Pulse.detach(); | |||
266 | for (unsigned i = 0; i < NumWorkers; i++) | |||
267 | V.push_back(std::thread(WorkerThread, std::ref(Cmd), &Counter, NumJobs, &HasErrors)); | |||
268 | for (auto &T : V) | |||
269 | T.join(); | |||
270 | return HasErrors ? 1 : 0; | |||
271 | } | |||
272 | ||||
273 | static void RssThread(Fuzzer *F, size_t RssLimitMb) { | |||
274 | while (true) { | |||
275 | SleepSeconds(1); | |||
276 | size_t Peak = GetPeakRSSMb(); | |||
277 | if (Peak > RssLimitMb) | |||
278 | F->RssLimitCallback(); | |||
279 | } | |||
280 | } | |||
281 | ||||
282 | static void StartRssThread(Fuzzer *F, size_t RssLimitMb) { | |||
283 | if (!RssLimitMb) return; | |||
284 | std::thread T(RssThread, F, RssLimitMb); | |||
285 | T.detach(); | |||
286 | } | |||
287 | ||||
288 | int RunOneTest(Fuzzer *F, const char *InputFilePath, size_t MaxLen) { | |||
289 | Unit U = FileToVector(InputFilePath); | |||
290 | if (MaxLen && MaxLen < U.size()) | |||
291 | U.resize(MaxLen); | |||
292 | F->ExecuteCallback(U.data(), U.size()); | |||
293 | F->TryDetectingAMemoryLeak(U.data(), U.size(), true); | |||
294 | return 0; | |||
295 | } | |||
296 | ||||
297 | static bool AllInputsAreFiles() { | |||
298 | if (Inputs->empty()) return false; | |||
299 | for (auto &Path : *Inputs) | |||
300 | if (!IsFile(Path)) | |||
301 | return false; | |||
302 | return true; | |||
303 | } | |||
304 | ||||
305 | static std::string GetDedupTokenFromFile(const std::string &Path) { | |||
306 | auto S = FileToString(Path); | |||
307 | auto Beg = S.find("DEDUP_TOKEN:"); | |||
308 | if (Beg == std::string::npos) | |||
309 | return ""; | |||
310 | auto End = S.find('\n', Beg); | |||
311 | if (End == std::string::npos) | |||
312 | return ""; | |||
313 | return S.substr(Beg, End - Beg); | |||
314 | } | |||
315 | ||||
316 | int CleanseCrashInput(const Vector<std::string> &Args, | |||
317 | const FuzzingOptions &Options) { | |||
318 | if (Inputs->size() != 1 || !Flags.exact_artifact_path) { | |||
319 | Printf("ERROR: -cleanse_crash should be given one input file and" | |||
320 | " -exact_artifact_path\n"); | |||
321 | exit(1); | |||
322 | } | |||
323 | std::string InputFilePath = Inputs->at(0); | |||
324 | std::string OutputFilePath = Flags.exact_artifact_path; | |||
325 | Command Cmd(Args); | |||
326 | Cmd.removeFlag("cleanse_crash"); | |||
327 | ||||
328 | assert(Cmd.hasArgument(InputFilePath))((Cmd.hasArgument(InputFilePath)) ? static_cast<void> ( 0) : __assert_fail ("Cmd.hasArgument(InputFilePath)", "/build/llvm-toolchain-snapshot-10~svn374877/projects/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 328, __PRETTY_FUNCTION__)); | |||
329 | Cmd.removeArgument(InputFilePath); | |||
330 | ||||
331 | auto LogFilePath = TempPath(".txt"); | |||
332 | auto TmpFilePath = TempPath(".repro"); | |||
333 | Cmd.addArgument(TmpFilePath); | |||
334 | Cmd.setOutputFile(LogFilePath); | |||
335 | Cmd.combineOutAndErr(); | |||
336 | ||||
337 | std::string CurrentFilePath = InputFilePath; | |||
338 | auto U = FileToVector(CurrentFilePath); | |||
339 | size_t Size = U.size(); | |||
340 | ||||
341 | const Vector<uint8_t> ReplacementBytes = {' ', 0xff}; | |||
342 | for (int NumAttempts = 0; NumAttempts < 5; NumAttempts++) { | |||
343 | bool Changed = false; | |||
344 | for (size_t Idx = 0; Idx < Size; Idx++) { | |||
345 | Printf("CLEANSE[%d]: Trying to replace byte %zd of %zd\n", NumAttempts, | |||
346 | Idx, Size); | |||
347 | uint8_t OriginalByte = U[Idx]; | |||
348 | if (ReplacementBytes.end() != std::find(ReplacementBytes.begin(), | |||
349 | ReplacementBytes.end(), | |||
350 | OriginalByte)) | |||
351 | continue; | |||
352 | for (auto NewByte : ReplacementBytes) { | |||
353 | U[Idx] = NewByte; | |||
354 | WriteToFile(U, TmpFilePath); | |||
355 | auto ExitCode = ExecuteCommand(Cmd); | |||
356 | RemoveFile(TmpFilePath); | |||
357 | if (!ExitCode) { | |||
358 | U[Idx] = OriginalByte; | |||
359 | } else { | |||
360 | Changed = true; | |||
361 | Printf("CLEANSE: Replaced byte %zd with 0x%x\n", Idx, NewByte); | |||
362 | WriteToFile(U, OutputFilePath); | |||
363 | break; | |||
364 | } | |||
365 | } | |||
366 | } | |||
367 | if (!Changed) break; | |||
368 | } | |||
369 | RemoveFile(LogFilePath); | |||
370 | return 0; | |||
371 | } | |||
372 | ||||
373 | int MinimizeCrashInput(const Vector<std::string> &Args, | |||
374 | const FuzzingOptions &Options) { | |||
375 | if (Inputs->size() != 1) { | |||
376 | Printf("ERROR: -minimize_crash should be given one input file\n"); | |||
377 | exit(1); | |||
378 | } | |||
379 | std::string InputFilePath = Inputs->at(0); | |||
380 | Command BaseCmd(Args); | |||
381 | BaseCmd.removeFlag("minimize_crash"); | |||
382 | BaseCmd.removeFlag("exact_artifact_path"); | |||
383 | assert(BaseCmd.hasArgument(InputFilePath))((BaseCmd.hasArgument(InputFilePath)) ? static_cast<void> (0) : __assert_fail ("BaseCmd.hasArgument(InputFilePath)", "/build/llvm-toolchain-snapshot-10~svn374877/projects/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 383, __PRETTY_FUNCTION__)); | |||
384 | BaseCmd.removeArgument(InputFilePath); | |||
385 | if (Flags.runs <= 0 && Flags.max_total_time == 0) { | |||
386 | Printf("INFO: you need to specify -runs=N or " | |||
387 | "-max_total_time=N with -minimize_crash=1\n" | |||
388 | "INFO: defaulting to -max_total_time=600\n"); | |||
389 | BaseCmd.addFlag("max_total_time", "600"); | |||
390 | } | |||
391 | ||||
392 | auto LogFilePath = TempPath(".txt"); | |||
393 | BaseCmd.setOutputFile(LogFilePath); | |||
394 | BaseCmd.combineOutAndErr(); | |||
395 | ||||
396 | std::string CurrentFilePath = InputFilePath; | |||
397 | while (true) { | |||
398 | Unit U = FileToVector(CurrentFilePath); | |||
399 | Printf("CRASH_MIN: minimizing crash input: '%s' (%zd bytes)\n", | |||
400 | CurrentFilePath.c_str(), U.size()); | |||
401 | ||||
402 | Command Cmd(BaseCmd); | |||
403 | Cmd.addArgument(CurrentFilePath); | |||
404 | ||||
405 | std::string CommandLine = Cmd.toString(); | |||
406 | Printf("CRASH_MIN: executing: %s\n", CommandLine.c_str()); | |||
407 | int ExitCode = ExecuteCommand(Cmd); | |||
408 | if (ExitCode == 0) { | |||
409 | Printf("ERROR: the input %s did not crash\n", CurrentFilePath.c_str()); | |||
410 | exit(1); | |||
411 | } | |||
412 | Printf("CRASH_MIN: '%s' (%zd bytes) caused a crash. Will try to minimize " | |||
413 | "it further\n", | |||
414 | CurrentFilePath.c_str(), U.size()); | |||
415 | auto DedupToken1 = GetDedupTokenFromFile(LogFilePath); | |||
416 | if (!DedupToken1.empty()) | |||
417 | Printf("CRASH_MIN: DedupToken1: %s\n", DedupToken1.c_str()); | |||
418 | ||||
419 | std::string ArtifactPath = | |||
420 | Flags.exact_artifact_path | |||
421 | ? Flags.exact_artifact_path | |||
422 | : Options.ArtifactPrefix + "minimized-from-" + Hash(U); | |||
423 | Cmd.addFlag("minimize_crash_internal_step", "1"); | |||
424 | Cmd.addFlag("exact_artifact_path", ArtifactPath); | |||
425 | CommandLine = Cmd.toString(); | |||
426 | Printf("CRASH_MIN: executing: %s\n", CommandLine.c_str()); | |||
427 | ExitCode = ExecuteCommand(Cmd); | |||
428 | CopyFileToErr(LogFilePath); | |||
429 | if (ExitCode == 0) { | |||
430 | if (Flags.exact_artifact_path) { | |||
431 | CurrentFilePath = Flags.exact_artifact_path; | |||
432 | WriteToFile(U, CurrentFilePath); | |||
433 | } | |||
434 | Printf("CRASH_MIN: failed to minimize beyond %s (%d bytes), exiting\n", | |||
435 | CurrentFilePath.c_str(), U.size()); | |||
436 | break; | |||
437 | } | |||
438 | auto DedupToken2 = GetDedupTokenFromFile(LogFilePath); | |||
439 | if (!DedupToken2.empty()) | |||
440 | Printf("CRASH_MIN: DedupToken2: %s\n", DedupToken2.c_str()); | |||
441 | ||||
442 | if (DedupToken1 != DedupToken2) { | |||
443 | if (Flags.exact_artifact_path) { | |||
444 | CurrentFilePath = Flags.exact_artifact_path; | |||
445 | WriteToFile(U, CurrentFilePath); | |||
446 | } | |||
447 | Printf("CRASH_MIN: mismatch in dedup tokens" | |||
448 | " (looks like a different bug). Won't minimize further\n"); | |||
449 | break; | |||
450 | } | |||
451 | ||||
452 | CurrentFilePath = ArtifactPath; | |||
453 | Printf("*********************************\n"); | |||
454 | } | |||
455 | RemoveFile(LogFilePath); | |||
456 | return 0; | |||
457 | } | |||
458 | ||||
459 | int MinimizeCrashInputInternalStep(Fuzzer *F, InputCorpus *Corpus) { | |||
460 | assert(Inputs->size() == 1)((Inputs->size() == 1) ? static_cast<void> (0) : __assert_fail ("Inputs->size() == 1", "/build/llvm-toolchain-snapshot-10~svn374877/projects/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 460, __PRETTY_FUNCTION__)); | |||
461 | std::string InputFilePath = Inputs->at(0); | |||
462 | Unit U = FileToVector(InputFilePath); | |||
463 | Printf("INFO: Starting MinimizeCrashInputInternalStep: %zd\n", U.size()); | |||
464 | if (U.size() < 2) { | |||
465 | Printf("INFO: The input is small enough, exiting\n"); | |||
466 | exit(0); | |||
467 | } | |||
468 | F->SetMaxInputLen(U.size()); | |||
469 | F->SetMaxMutationLen(U.size() - 1); | |||
470 | F->MinimizeCrashLoop(U); | |||
471 | Printf("INFO: Done MinimizeCrashInputInternalStep, no crashes found\n"); | |||
472 | exit(0); | |||
473 | return 0; | |||
474 | } | |||
475 | ||||
476 | void Merge(Fuzzer *F, FuzzingOptions &Options, const Vector<std::string> &Args, | |||
477 | const Vector<std::string> &Corpora, const char *CFPathOrNull) { | |||
478 | if (Corpora.size() < 2) { | |||
479 | Printf("INFO: Merge requires two or more corpus dirs\n"); | |||
480 | exit(0); | |||
481 | } | |||
482 | ||||
483 | Vector<SizedFile> OldCorpus, NewCorpus; | |||
484 | GetSizedFilesFromDir(Corpora[0], &OldCorpus); | |||
485 | for (size_t i = 1; i < Corpora.size(); i++) | |||
486 | GetSizedFilesFromDir(Corpora[i], &NewCorpus); | |||
487 | std::sort(OldCorpus.begin(), OldCorpus.end()); | |||
488 | std::sort(NewCorpus.begin(), NewCorpus.end()); | |||
489 | ||||
490 | std::string CFPath = CFPathOrNull ? CFPathOrNull : TempPath(".txt"); | |||
491 | Vector<std::string> NewFiles; | |||
492 | Set<uint32_t> NewFeatures, NewCov; | |||
493 | CrashResistantMerge(Args, OldCorpus, NewCorpus, &NewFiles, {}, &NewFeatures, | |||
494 | {}, &NewCov, CFPath, true); | |||
495 | for (auto &Path : NewFiles) | |||
496 | F->WriteToOutputCorpus(FileToVector(Path, Options.MaxLen)); | |||
497 | // We are done, delete the control file if it was a temporary one. | |||
498 | if (!Flags.merge_control_file) | |||
499 | RemoveFile(CFPath); | |||
500 | ||||
501 | exit(0); | |||
502 | } | |||
503 | ||||
504 | int AnalyzeDictionary(Fuzzer *F, const Vector<Unit>& Dict, | |||
505 | UnitVector& Corpus) { | |||
506 | Printf("Started dictionary minimization (up to %d tests)\n", | |||
507 | Dict.size() * Corpus.size() * 2); | |||
508 | ||||
509 | // Scores and usage count for each dictionary unit. | |||
510 | Vector<int> Scores(Dict.size()); | |||
511 | Vector<int> Usages(Dict.size()); | |||
512 | ||||
513 | Vector<size_t> InitialFeatures; | |||
514 | Vector<size_t> ModifiedFeatures; | |||
515 | for (auto &C : Corpus) { | |||
516 | // Get coverage for the testcase without modifications. | |||
517 | F->ExecuteCallback(C.data(), C.size()); | |||
518 | InitialFeatures.clear(); | |||
519 | TPC.CollectFeatures([&](size_t Feature) { | |||
520 | InitialFeatures.push_back(Feature); | |||
521 | }); | |||
522 | ||||
523 | for (size_t i = 0; i < Dict.size(); ++i) { | |||
524 | Vector<uint8_t> Data = C; | |||
525 | auto StartPos = std::search(Data.begin(), Data.end(), | |||
526 | Dict[i].begin(), Dict[i].end()); | |||
527 | // Skip dictionary unit, if the testcase does not contain it. | |||
528 | if (StartPos == Data.end()) | |||
529 | continue; | |||
530 | ||||
531 | ++Usages[i]; | |||
532 | while (StartPos != Data.end()) { | |||
533 | // Replace all occurrences of dictionary unit in the testcase. | |||
534 | auto EndPos = StartPos + Dict[i].size(); | |||
535 | for (auto It = StartPos; It != EndPos; ++It) | |||
536 | *It ^= 0xFF; | |||
537 | ||||
538 | StartPos = std::search(EndPos, Data.end(), | |||
539 | Dict[i].begin(), Dict[i].end()); | |||
540 | } | |||
541 | ||||
542 | // Get coverage for testcase with masked occurrences of dictionary unit. | |||
543 | F->ExecuteCallback(Data.data(), Data.size()); | |||
544 | ModifiedFeatures.clear(); | |||
545 | TPC.CollectFeatures([&](size_t Feature) { | |||
546 | ModifiedFeatures.push_back(Feature); | |||
547 | }); | |||
548 | ||||
549 | if (InitialFeatures == ModifiedFeatures) | |||
550 | --Scores[i]; | |||
551 | else | |||
552 | Scores[i] += 2; | |||
553 | } | |||
554 | } | |||
555 | ||||
556 | Printf("###### Useless dictionary elements. ######\n"); | |||
557 | for (size_t i = 0; i < Dict.size(); ++i) { | |||
558 | // Dictionary units with positive score are treated as useful ones. | |||
559 | if (Scores[i] > 0) | |||
560 | continue; | |||
561 | ||||
562 | Printf("\""); | |||
563 | PrintASCII(Dict[i].data(), Dict[i].size(), "\""); | |||
564 | Printf(" # Score: %d, Used: %d\n", Scores[i], Usages[i]); | |||
565 | } | |||
566 | Printf("###### End of useless dictionary elements. ######\n"); | |||
567 | return 0; | |||
568 | } | |||
569 | ||||
570 | Vector<std::string> ParseSeedInuts(const char *seed_inputs) { | |||
571 | // Parse -seed_inputs=file1,file2,... or -seed_inputs=@seed_inputs_file | |||
572 | Vector<std::string> Files; | |||
573 | if (!seed_inputs) return Files; | |||
574 | std::string SeedInputs; | |||
575 | if (Flags.seed_inputs[0] == '@') | |||
576 | SeedInputs = FileToString(Flags.seed_inputs + 1); // File contains list. | |||
577 | else | |||
578 | SeedInputs = Flags.seed_inputs; // seed_inputs contains the list. | |||
579 | if (SeedInputs.empty()) { | |||
580 | Printf("seed_inputs is empty or @file does not exist.\n"); | |||
581 | exit(1); | |||
582 | } | |||
583 | // Parse SeedInputs. | |||
584 | size_t comma_pos = 0; | |||
585 | while ((comma_pos = SeedInputs.find_last_of(',')) != std::string::npos) { | |||
586 | Files.push_back(SeedInputs.substr(comma_pos + 1)); | |||
587 | SeedInputs = SeedInputs.substr(0, comma_pos); | |||
588 | } | |||
589 | Files.push_back(SeedInputs); | |||
590 | return Files; | |||
591 | } | |||
592 | ||||
593 | static Vector<SizedFile> ReadCorpora(const Vector<std::string> &CorpusDirs, | |||
594 | const Vector<std::string> &ExtraSeedFiles) { | |||
595 | Vector<SizedFile> SizedFiles; | |||
596 | size_t LastNumFiles = 0; | |||
597 | for (auto &Dir : CorpusDirs) { | |||
598 | GetSizedFilesFromDir(Dir, &SizedFiles); | |||
599 | Printf("INFO: % 8zd files found in %s\n", SizedFiles.size() - LastNumFiles, | |||
600 | Dir.c_str()); | |||
601 | LastNumFiles = SizedFiles.size(); | |||
602 | } | |||
603 | for (auto &File : ExtraSeedFiles) | |||
604 | if (auto Size = FileSize(File)) | |||
605 | SizedFiles.push_back({File, Size}); | |||
606 | return SizedFiles; | |||
607 | } | |||
608 | ||||
609 | int FuzzerDriver(int *argc, char ***argv, UserCallback Callback) { | |||
610 | using namespace fuzzer; | |||
611 | assert(argc && argv && "Argument pointers cannot be nullptr")((argc && argv && "Argument pointers cannot be nullptr" ) ? static_cast<void> (0) : __assert_fail ("argc && argv && \"Argument pointers cannot be nullptr\"" , "/build/llvm-toolchain-snapshot-10~svn374877/projects/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 611, __PRETTY_FUNCTION__)); | |||
| ||||
612 | std::string Argv0((*argv)[0]); | |||
613 | EF = new ExternalFunctions(); | |||
614 | if (EF->LLVMFuzzerInitialize) | |||
615 | EF->LLVMFuzzerInitialize(argc, argv); | |||
616 | if (EF->__msan_scoped_disable_interceptor_checks) | |||
617 | EF->__msan_scoped_disable_interceptor_checks(); | |||
618 | const Vector<std::string> Args(*argv, *argv + *argc); | |||
619 | assert(!Args.empty())((!Args.empty()) ? static_cast<void> (0) : __assert_fail ("!Args.empty()", "/build/llvm-toolchain-snapshot-10~svn374877/projects/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 619, __PRETTY_FUNCTION__)); | |||
620 | ProgName = new std::string(Args[0]); | |||
621 | if (Argv0 != *ProgName) { | |||
622 | Printf("ERROR: argv[0] has been modified in LLVMFuzzerInitialize\n"); | |||
623 | exit(1); | |||
624 | } | |||
625 | ParseFlags(Args, EF); | |||
626 | if (Flags.help) { | |||
627 | PrintHelp(); | |||
628 | return 0; | |||
629 | } | |||
630 | ||||
631 | if (Flags.close_fd_mask & 2) | |||
632 | DupAndCloseStderr(); | |||
633 | if (Flags.close_fd_mask & 1) | |||
634 | CloseStdout(); | |||
635 | ||||
636 | if (Flags.jobs > 0 && Flags.workers == 0) { | |||
637 | Flags.workers = std::min(NumberOfCpuCores() / 2, Flags.jobs); | |||
638 | if (Flags.workers > 1) | |||
639 | Printf("Running %u workers\n", Flags.workers); | |||
640 | } | |||
641 | ||||
642 | if (Flags.workers > 0 && Flags.jobs > 0) | |||
643 | return RunInMultipleProcesses(Args, Flags.workers, Flags.jobs); | |||
644 | ||||
645 | FuzzingOptions Options; | |||
646 | Options.Verbosity = Flags.verbosity; | |||
647 | Options.MaxLen = Flags.max_len; | |||
648 | Options.LenControl = Flags.len_control; | |||
649 | Options.UnitTimeoutSec = Flags.timeout; | |||
650 | Options.ErrorExitCode = Flags.error_exitcode; | |||
651 | Options.TimeoutExitCode = Flags.timeout_exitcode; | |||
652 | Options.IgnoreTimeouts = Flags.ignore_timeouts; | |||
653 | Options.IgnoreOOMs = Flags.ignore_ooms; | |||
654 | Options.IgnoreCrashes = Flags.ignore_crashes; | |||
655 | Options.MaxTotalTimeSec = Flags.max_total_time; | |||
656 | Options.DoCrossOver = Flags.cross_over; | |||
657 | Options.MutateDepth = Flags.mutate_depth; | |||
658 | Options.ReduceDepth = Flags.reduce_depth; | |||
659 | Options.UseCounters = Flags.use_counters; | |||
660 | Options.UseMemmem = Flags.use_memmem; | |||
661 | Options.UseCmp = Flags.use_cmp; | |||
662 | Options.UseValueProfile = Flags.use_value_profile; | |||
663 | Options.Shrink = Flags.shrink; | |||
664 | Options.ReduceInputs = Flags.reduce_inputs; | |||
665 | Options.ShuffleAtStartUp = Flags.shuffle; | |||
666 | Options.PreferSmall = Flags.prefer_small; | |||
667 | Options.ReloadIntervalSec = Flags.reload; | |||
668 | Options.OnlyASCII = Flags.only_ascii; | |||
669 | Options.DetectLeaks = Flags.detect_leaks; | |||
670 | Options.PurgeAllocatorIntervalSec = Flags.purge_allocator_interval; | |||
671 | Options.TraceMalloc = Flags.trace_malloc; | |||
672 | Options.RssLimitMb = Flags.rss_limit_mb; | |||
673 | Options.MallocLimitMb = Flags.malloc_limit_mb; | |||
674 | if (!Options.MallocLimitMb) | |||
675 | Options.MallocLimitMb = Options.RssLimitMb; | |||
676 | if (Flags.runs >= 0) | |||
677 | Options.MaxNumberOfRuns = Flags.runs; | |||
678 | if (!Inputs->empty() && !Flags.minimize_crash_internal_step) | |||
679 | Options.OutputCorpus = (*Inputs)[0]; | |||
680 | Options.ReportSlowUnits = Flags.report_slow_units; | |||
681 | if (Flags.artifact_prefix) | |||
682 | Options.ArtifactPrefix = Flags.artifact_prefix; | |||
683 | if (Flags.exact_artifact_path) | |||
684 | Options.ExactArtifactPath = Flags.exact_artifact_path; | |||
685 | Vector<Unit> Dictionary; | |||
686 | if (Flags.dict) | |||
687 | if (!ParseDictionaryFile(FileToString(Flags.dict), &Dictionary)) | |||
688 | return 1; | |||
689 | if (Flags.verbosity > 0 && !Dictionary.empty()) | |||
690 | Printf("Dictionary: %zd entries\n", Dictionary.size()); | |||
691 | bool RunIndividualFiles = AllInputsAreFiles(); | |||
692 | Options.SaveArtifacts = | |||
693 | !RunIndividualFiles
| |||
694 | Options.PrintNewCovPcs = Flags.print_pcs; | |||
695 | Options.PrintNewCovFuncs = Flags.print_funcs; | |||
696 | Options.PrintFinalStats = Flags.print_final_stats; | |||
697 | Options.PrintCorpusStats = Flags.print_corpus_stats; | |||
698 | Options.PrintCoverage = Flags.print_coverage; | |||
699 | if (Flags.exit_on_src_pos) | |||
700 | Options.ExitOnSrcPos = Flags.exit_on_src_pos; | |||
701 | if (Flags.exit_on_item) | |||
702 | Options.ExitOnItem = Flags.exit_on_item; | |||
703 | if (Flags.focus_function) | |||
704 | Options.FocusFunction = Flags.focus_function; | |||
705 | if (Flags.data_flow_trace) | |||
706 | Options.DataFlowTrace = Flags.data_flow_trace; | |||
707 | if (Flags.features_dir) | |||
708 | Options.FeaturesDir = Flags.features_dir; | |||
709 | if (Flags.collect_data_flow) | |||
710 | Options.CollectDataFlow = Flags.collect_data_flow; | |||
711 | if (Flags.stop_file) | |||
712 | Options.StopFile = Flags.stop_file; | |||
713 | ||||
714 | unsigned Seed = Flags.seed; | |||
715 | // Initialize Seed. | |||
716 | if (Seed == 0) | |||
717 | Seed = | |||
718 | std::chrono::system_clock::now().time_since_epoch().count() + GetPid(); | |||
719 | if (Flags.verbosity) | |||
720 | Printf("INFO: Seed: %u\n", Seed); | |||
721 | ||||
722 | if (Flags.collect_data_flow
| |||
723 | if (RunIndividualFiles) | |||
724 | return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, | |||
725 | ReadCorpora({}, *Inputs)); | |||
726 | else | |||
727 | return CollectDataFlow(Flags.collect_data_flow, Flags.data_flow_trace, | |||
728 | ReadCorpora(*Inputs, {})); | |||
729 | } | |||
730 | ||||
731 | Random Rand(Seed); | |||
732 | auto *MD = new MutationDispatcher(Rand, Options); | |||
733 | auto *Corpus = new InputCorpus(Options.OutputCorpus); | |||
734 | auto *F = new Fuzzer(Callback, *Corpus, *MD, Options); | |||
735 | ||||
736 | for (auto &U: Dictionary) | |||
737 | if (U.size() <= Word::GetMaxSize()) | |||
738 | MD->AddWordToManualDictionary(Word(U.data(), U.size())); | |||
739 | ||||
740 | StartRssThread(F, Flags.rss_limit_mb); | |||
741 | ||||
742 | Options.HandleAbrt = Flags.handle_abrt; | |||
743 | Options.HandleBus = Flags.handle_bus; | |||
744 | Options.HandleFpe = Flags.handle_fpe; | |||
745 | Options.HandleIll = Flags.handle_ill; | |||
746 | Options.HandleInt = Flags.handle_int; | |||
747 | Options.HandleSegv = Flags.handle_segv; | |||
748 | Options.HandleTerm = Flags.handle_term; | |||
749 | Options.HandleXfsz = Flags.handle_xfsz; | |||
750 | Options.HandleUsr1 = Flags.handle_usr1; | |||
751 | Options.HandleUsr2 = Flags.handle_usr2; | |||
752 | SetSignalHandler(Options); | |||
753 | ||||
754 | std::atexit(Fuzzer::StaticExitCallback); | |||
755 | ||||
756 | if (Flags.minimize_crash) | |||
757 | return MinimizeCrashInput(Args, Options); | |||
| ||||
758 | ||||
759 | if (Flags.minimize_crash_internal_step) | |||
760 | return MinimizeCrashInputInternalStep(F, Corpus); | |||
761 | ||||
762 | if (Flags.cleanse_crash) | |||
763 | return CleanseCrashInput(Args, Options); | |||
764 | ||||
765 | if (RunIndividualFiles) { | |||
766 | Options.SaveArtifacts = false; | |||
767 | int Runs = std::max(1, Flags.runs); | |||
768 | Printf("%s: Running %zd inputs %d time(s) each.\n", ProgName->c_str(), | |||
769 | Inputs->size(), Runs); | |||
770 | for (auto &Path : *Inputs) { | |||
771 | auto StartTime = system_clock::now(); | |||
772 | Printf("Running: %s\n", Path.c_str()); | |||
773 | for (int Iter = 0; Iter < Runs; Iter++) | |||
774 | RunOneTest(F, Path.c_str(), Options.MaxLen); | |||
775 | auto StopTime = system_clock::now(); | |||
776 | auto MS = duration_cast<milliseconds>(StopTime - StartTime).count(); | |||
777 | Printf("Executed %s in %zd ms\n", Path.c_str(), (long)MS); | |||
778 | } | |||
779 | Printf("***\n" | |||
780 | "*** NOTE: fuzzing was not performed, you have only\n" | |||
781 | "*** executed the target code on a fixed set of inputs.\n" | |||
782 | "***\n"); | |||
783 | F->PrintFinalStats(); | |||
784 | exit(0); | |||
785 | } | |||
786 | ||||
787 | if (Flags.fork) | |||
788 | FuzzWithFork(F->GetMD().GetRand(), Options, Args, *Inputs, Flags.fork); | |||
789 | ||||
790 | if (Flags.merge) | |||
791 | Merge(F, Options, Args, *Inputs, Flags.merge_control_file); | |||
792 | ||||
793 | if (Flags.merge_inner) { | |||
794 | const size_t kDefaultMaxMergeLen = 1 << 20; | |||
795 | if (Options.MaxLen == 0) | |||
796 | F->SetMaxInputLen(kDefaultMaxMergeLen); | |||
797 | assert(Flags.merge_control_file)((Flags.merge_control_file) ? static_cast<void> (0) : __assert_fail ("Flags.merge_control_file", "/build/llvm-toolchain-snapshot-10~svn374877/projects/compiler-rt/lib/fuzzer/FuzzerDriver.cpp" , 797, __PRETTY_FUNCTION__)); | |||
798 | F->CrashResistantMergeInternalStep(Flags.merge_control_file); | |||
799 | exit(0); | |||
800 | } | |||
801 | ||||
802 | if (Flags.analyze_dict) { | |||
803 | size_t MaxLen = INT_MAX2147483647; // Large max length. | |||
804 | UnitVector InitialCorpus; | |||
805 | for (auto &Inp : *Inputs) { | |||
806 | Printf("Loading corpus dir: %s\n", Inp.c_str()); | |||
807 | ReadDirToVectorOfUnits(Inp.c_str(), &InitialCorpus, nullptr, | |||
808 | MaxLen, /*ExitOnError=*/false); | |||
809 | } | |||
810 | ||||
811 | if (Dictionary.empty() || Inputs->empty()) { | |||
812 | Printf("ERROR: can't analyze dict without dict and corpus provided\n"); | |||
813 | return 1; | |||
814 | } | |||
815 | if (AnalyzeDictionary(F, Dictionary, InitialCorpus)) { | |||
816 | Printf("Dictionary analysis failed\n"); | |||
817 | exit(1); | |||
818 | } | |||
819 | Printf("Dictionary analysis succeeded\n"); | |||
820 | exit(0); | |||
821 | } | |||
822 | ||||
823 | auto CorporaFiles = ReadCorpora(*Inputs, ParseSeedInuts(Flags.seed_inputs)); | |||
824 | F->Loop(CorporaFiles); | |||
825 | ||||
826 | if (Flags.verbosity) | |||
827 | Printf("Done %zd runs in %zd second(s)\n", F->getTotalNumberOfRuns(), | |||
828 | F->secondsSinceProcessStartUp()); | |||
829 | F->PrintFinalStats(); | |||
830 | ||||
831 | exit(0); // Don't let F destroy itself. | |||
832 | } | |||
833 | ||||
834 | // Storage for global ExternalFunctions object. | |||
835 | ExternalFunctions *EF = nullptr; | |||
836 | ||||
837 | } // namespace fuzzer |