Bug Summary

File:clang/lib/Tooling/JSONCompilationDatabase.cpp
Warning:line 383, column 31
Dereference of null pointer (loaded from variable 'SequenceString')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name JSONCompilationDatabase.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -relaxed-aliasing -fmath-errno -fno-rounding-math -mconstructor-aliases -munwind-tables -target-cpu x86-64 -tune-cpu generic -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-12/lib/clang/12.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/build-llvm/tools/clang/lib/Tooling -I /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/clang/lib/Tooling -I /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/clang/include -I /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/build-llvm/include -I /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-12/lib/clang/12.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/build-llvm/tools/clang/lib/Tooling -fdebug-prefix-map=/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b=. -ferror-limit 19 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-09-17-195756-12974-1 -x c++ /build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/clang/lib/Tooling/JSONCompilationDatabase.cpp

/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/clang/lib/Tooling/JSONCompilationDatabase.cpp

→
1//===- JSONCompilationDatabase.cpp ----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the JSONCompilationDatabase.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Tooling/JSONCompilationDatabase.h"
14#include "clang/Basic/LLVM.h"
15#include "clang/Tooling/CompilationDatabase.h"
16#include "clang/Tooling/CompilationDatabasePluginRegistry.h"
17#include "clang/Tooling/Tooling.h"
18#include "llvm/ADT/Optional.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallString.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/ADT/Triple.h"
24#include "llvm/Support/Allocator.h"
25#include "llvm/Support/Casting.h"
26#include "llvm/Support/CommandLine.h"
27#include "llvm/Support/ErrorOr.h"
28#include "llvm/Support/Host.h"
29#include "llvm/Support/MemoryBuffer.h"
30#include "llvm/Support/Path.h"
31#include "llvm/Support/StringSaver.h"
32#include "llvm/Support/VirtualFileSystem.h"
33#include "llvm/Support/YAMLParser.h"
34#include "llvm/Support/raw_ostream.h"
35#include <cassert>
36#include <memory>
37#include <string>
38#include <system_error>
39#include <tuple>
40#include <utility>
41#include <vector>
42
43using namespace clang;
44using namespace tooling;
45
46namespace {
47
48/// A parser for escaped strings of command line arguments.
49///
50/// Assumes \-escaping for quoted arguments (see the documentation of
51/// unescapeCommandLine(...)).
52class CommandLineArgumentParser {
53 public:
54 CommandLineArgumentParser(StringRef CommandLine)
55 : Input(CommandLine), Position(Input.begin()-1) {}
56
57 std::vector<std::string> parse() {
58 bool HasMoreInput = true;
59 while (HasMoreInput && nextNonWhitespace()) {
60 std::string Argument;
61 HasMoreInput = parseStringInto(Argument);
62 CommandLine.push_back(Argument);
63 }
64 return CommandLine;
65 }
66
67 private:
68 // All private methods return true if there is more input available.
69
70 bool parseStringInto(std::string &String) {
71 do {
72 if (*Position == '"') {
73 if (!parseDoubleQuotedStringInto(String)) return false;
74 } else if (*Position == '\'') {
75 if (!parseSingleQuotedStringInto(String)) return false;
76 } else {
77 if (!parseFreeStringInto(String)) return false;
78 }
79 } while (*Position != ' ');
80 return true;
81 }
82
83 bool parseDoubleQuotedStringInto(std::string &String) {
84 if (!next()) return false;
85 while (*Position != '"') {
86 if (!skipEscapeCharacter()) return false;
87 String.push_back(*Position);
88 if (!next()) return false;
89 }
90 return next();
91 }
92
93 bool parseSingleQuotedStringInto(std::string &String) {
94 if (!next()) return false;
95 while (*Position != '\'') {
96 String.push_back(*Position);
97 if (!next()) return false;
98 }
99 return next();
100 }
101
102 bool parseFreeStringInto(std::string &String) {
103 do {
104 if (!skipEscapeCharacter()) return false;
105 String.push_back(*Position);
106 if (!next()) return false;
107 } while (*Position != ' ' && *Position != '"' && *Position != '\'');
108 return true;
109 }
110
111 bool skipEscapeCharacter() {
112 if (*Position == '\\') {
113 return next();
114 }
115 return true;
116 }
117
118 bool nextNonWhitespace() {
119 do {
120 if (!next()) return false;
121 } while (*Position == ' ');
122 return true;
123 }
124
125 bool next() {
126 ++Position;
127 return Position != Input.end();
128 }
129
130 const StringRef Input;
131 StringRef::iterator Position;
132 std::vector<std::string> CommandLine;
133};
134
135std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
136 StringRef EscapedCommandLine) {
137 if (Syntax == JSONCommandLineSyntax::AutoDetect) {
138 Syntax = JSONCommandLineSyntax::Gnu;
139 llvm::Triple Triple(llvm::sys::getProcessTriple());
140 if (Triple.getOS() == llvm::Triple::OSType::Win32) {
141 // Assume Windows command line parsing on Win32 unless the triple
142 // explicitly tells us otherwise.
143 if (!Triple.hasEnvironment() ||
144 Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC)
145 Syntax = JSONCommandLineSyntax::Windows;
146 }
147 }
148
149 if (Syntax == JSONCommandLineSyntax::Windows) {
150 llvm::BumpPtrAllocator Alloc;
151 llvm::StringSaver Saver(Alloc);
152 llvm::SmallVector<const char *, 64> T;
153 llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T);
154 std::vector<std::string> Result(T.begin(), T.end());
155 return Result;
156 }
157 assert(Syntax == JSONCommandLineSyntax::Gnu)((Syntax == JSONCommandLineSyntax::Gnu) ? static_cast<void
> (0) : __assert_fail ("Syntax == JSONCommandLineSyntax::Gnu"
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/clang/lib/Tooling/JSONCompilationDatabase.cpp"
, 157, __PRETTY_FUNCTION__));
158 CommandLineArgumentParser parser(EscapedCommandLine);
159 return parser.parse();
160}
161
162// This plugin locates a nearby compile_command.json file, and also infers
163// compile commands for files not present in the database.
164class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
165 std::unique_ptr<CompilationDatabase>
166 loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
167 SmallString<1024> JSONDatabasePath(Directory);
168 llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
169 auto Base = JSONCompilationDatabase::loadFromFile(
170 JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect);
171 return Base ? inferTargetAndDriverMode(
172 inferMissingCompileCommands(expandResponseFiles(
173 std::move(Base), llvm::vfs::getRealFileSystem())))
174 : nullptr;
175 }
176};
177
178} // namespace
179
180// Register the JSONCompilationDatabasePlugin with the
181// CompilationDatabasePluginRegistry using this statically initialized variable.
182static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>
183X("json-compilation-database", "Reads JSON formatted compilation databases");
184
185namespace clang {
186namespace tooling {
187
188// This anchor is used to force the linker to link in the generated object file
189// and thus register the JSONCompilationDatabasePlugin.
190volatile int JSONAnchorSource = 0;
191
192} // namespace tooling
193} // namespace clang
194
195std::unique_ptr<JSONCompilationDatabase>
196JSONCompilationDatabase::loadFromFile(StringRef FilePath,
197 std::string &ErrorMessage,
198 JSONCommandLineSyntax Syntax) {
199 // Don't mmap: if we're a long-lived process, the build system may overwrite.
200 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
201 llvm::MemoryBuffer::getFile(FilePath, /*FileSize=*/-1,
202 /*RequiresNullTerminator=*/true,
203 /*IsVolatile=*/true);
204 if (std::error_code Result = DatabaseBuffer.getError()) {
205 ErrorMessage = "Error while opening JSON database: " + Result.message();
206 return nullptr;
207 }
208 std::unique_ptr<JSONCompilationDatabase> Database(
209 new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax));
210 if (!Database->parse(ErrorMessage))
211 return nullptr;
212 return Database;
213}
214
215std::unique_ptr<JSONCompilationDatabase>
216JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString,
217 std::string &ErrorMessage,
218 JSONCommandLineSyntax Syntax) {
219 std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
220 llvm::MemoryBuffer::getMemBuffer(DatabaseString));
221 std::unique_ptr<JSONCompilationDatabase> Database(
222 new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
223 if (!Database->parse(ErrorMessage))
1
Calling 'JSONCompilationDatabase::parse'
→
224 return nullptr;
225 return Database;
226}
227
228std::vector<CompileCommand>
229JSONCompilationDatabase::getCompileCommands(StringRef FilePath) const {
230 SmallString<128> NativeFilePath;
231 llvm::sys::path::native(FilePath, NativeFilePath);
232
233 std::string Error;
234 llvm::raw_string_ostream ES(Error);
235 StringRef Match = MatchTrie.findEquivalent(NativeFilePath, ES);
236 if (Match.empty())
237 return {};
238 const auto CommandsRefI = IndexByFile.find(Match);
239 if (CommandsRefI == IndexByFile.end())
240 return {};
241 std::vector<CompileCommand> Commands;
242 getCommands(CommandsRefI->getValue(), Commands);
243 return Commands;
244}
245
246std::vector<std::string>
247JSONCompilationDatabase::getAllFiles() const {
248 std::vector<std::string> Result;
249 for (const auto &CommandRef : IndexByFile)
250 Result.push_back(CommandRef.first().str());
251 return Result;
252}
253
254std::vector<CompileCommand>
255JSONCompilationDatabase::getAllCompileCommands() const {
256 std::vector<CompileCommand> Commands;
257 getCommands(AllCommands, Commands);
258 return Commands;
259}
260
261static llvm::StringRef stripExecutableExtension(llvm::StringRef Name) {
262 Name.consume_back(".exe");
263 return Name;
264}
265
266// There are compiler-wrappers (ccache, distcc, gomacc) that take the "real"
267// compiler as an argument, e.g. distcc gcc -O3 foo.c.
268// These end up in compile_commands.json when people set CC="distcc gcc".
269// Clang's driver doesn't understand this, so we need to unwrap.
270static bool unwrapCommand(std::vector<std::string> &Args) {
271 if (Args.size() < 2)
272 return false;
273 StringRef Wrapper =
274 stripExecutableExtension(llvm::sys::path::filename(Args.front()));
275 if (Wrapper == "distcc" || Wrapper == "gomacc" || Wrapper == "ccache") {
276 // Most of these wrappers support being invoked 3 ways:
277 // `distcc g++ file.c` This is the mode we're trying to match.
278 // We need to drop `distcc`.
279 // `distcc file.c` This acts like compiler is cc or similar.
280 // Clang's driver can handle this, no change needed.
281 // `g++ file.c` g++ is a symlink to distcc.
282 // We don't even notice this case, and all is well.
283 //
284 // We need to distinguish between the first and second case.
285 // The wrappers themselves don't take flags, so Args[1] is a compiler flag,
286 // an input file, or a compiler. Inputs have extensions, compilers don't.
287 bool HasCompiler =
288 (Args[1][0] != '-') &&
289 !llvm::sys::path::has_extension(stripExecutableExtension(Args[1]));
290 if (HasCompiler) {
291 Args.erase(Args.begin());
292 return true;
293 }
294 // If !HasCompiler, wrappers act like GCC. Fine: so do we.
295 }
296 return false;
297}
298
299static std::vector<std::string>
300nodeToCommandLine(JSONCommandLineSyntax Syntax,
301 const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
302 SmallString<1024> Storage;
303 std::vector<std::string> Arguments;
304 if (Nodes.size() == 1)
305 Arguments = unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage));
306 else
307 for (const auto *Node : Nodes)
308 Arguments.push_back(std::string(Node->getValue(Storage)));
309 // There may be multiple wrappers: using distcc and ccache together is common.
310 while (unwrapCommand(Arguments))
311 ;
312 return Arguments;
313}
314
315void JSONCompilationDatabase::getCommands(
316 ArrayRef<CompileCommandRef> CommandsRef,
317 std::vector<CompileCommand> &Commands) const {
318 for (const auto &CommandRef : CommandsRef) {
319 SmallString<8> DirectoryStorage;
320 SmallString<32> FilenameStorage;
321 SmallString<32> OutputStorage;
322 auto Output = std::get<3>(CommandRef);
323 Commands.emplace_back(
324 std::get<0>(CommandRef)->getValue(DirectoryStorage),
325 std::get<1>(CommandRef)->getValue(FilenameStorage),
326 nodeToCommandLine(Syntax, std::get<2>(CommandRef)),
327 Output ? Output->getValue(OutputStorage) : "");
328 }
329}
330
331bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
332 llvm::yaml::document_iterator I = YAMLStream.begin();
333 if (I == YAMLStream.end()) {
2
←
Calling 'document_iterator::operator=='
→
14
←
Returning from 'document_iterator::operator=='
→
15
←
Taking false branch
→
334 ErrorMessage = "Error while parsing YAML.";
335 return false;
336 }
337 llvm::yaml::Node *Root = I->getRoot();
16
←
Calling 'Document::getRoot'
→
20
←
Returning from 'Document::getRoot'
→
338 if (!Root
20.1
'Root' is non-null
20.1
'Root' is non-null
) {
21
←
Taking false branch
→
339 ErrorMessage = "Error while parsing YAML.";
340 return false;
341 }
342 auto *Array = dyn_cast<llvm::yaml::SequenceNode>(Root);
22
←
Assuming 'Root' is a 'SequenceNode'
→
343 if (!Array
22.1
'Array' is non-null
22.1
'Array' is non-null
) {
23
←
Taking false branch
→
344 ErrorMessage = "Expected array.";
345 return false;
346 }
347 for (auto &NextObject : *Array) {
348 auto *Object = dyn_cast<llvm::yaml::MappingNode>(&NextObject);
24
←
Assuming the object is a 'MappingNode'
→
349 if (!Object
24.1
'Object' is non-null
24.1
'Object' is non-null
) {
25
←
Taking false branch
→
350 ErrorMessage = "Expected object.";
351 return false;
352 }
353 llvm::yaml::ScalarNode *Directory = nullptr;
354 llvm::Optional<std::vector<llvm::yaml::ScalarNode *>> Command;
355 llvm::yaml::ScalarNode *File = nullptr;
356 llvm::yaml::ScalarNode *Output = nullptr;
357 for (auto& NextKeyValue : *Object) {
358 auto *KeyString = dyn_cast<llvm::yaml::ScalarNode>(NextKeyValue.getKey());
26
←
Assuming the object is a 'ScalarNode'
→
359 if (!KeyString
26.1
'KeyString' is non-null
26.1
'KeyString' is non-null
) {
27
←
Taking false branch
→
360 ErrorMessage = "Expected strings as key.";
361 return false;
362 }
363 SmallString<10> KeyStorage;
364 StringRef KeyValue = KeyString->getValue(KeyStorage);
365 llvm::yaml::Node *Value = NextKeyValue.getValue();
366 if (!Value) {
28
←
Assuming 'Value' is non-null
→
29
←
Taking false branch
→
367 ErrorMessage = "Expected value.";
368 return false;
369 }
370 auto *ValueString = dyn_cast<llvm::yaml::ScalarNode>(Value);
30
←
Assuming 'Value' is a 'ScalarNode'
→
371 auto *SequenceString = dyn_cast<llvm::yaml::SequenceNode>(Value);
31
←
Assuming 'Value' is not a 'SequenceNode'
→
32
←
'SequenceString' initialized to a null pointer value
→
372 if (KeyValue == "arguments" && !SequenceString) {
33
←
Assuming the condition is false
→
34
←
Taking false branch
→
373 ErrorMessage = "Expected sequence as value.";
374 return false;
375 } else if (KeyValue != "arguments" && !ValueString) {
35
←
Taking false branch
→
376 ErrorMessage = "Expected string as value.";
377 return false;
378 }
379 if (KeyValue == "directory") {
36
←
Assuming the condition is false
→
37
←
Taking false branch
→
380 Directory = ValueString;
381 } else if (KeyValue == "arguments") {
38
←
Assuming the condition is true
→
39
←
Taking true branch
→
382 Command = std::vector<llvm::yaml::ScalarNode *>();
383 for (auto &Argument : *SequenceString) {
40
←
Dereference of null pointer (loaded from variable 'SequenceString')
384 auto *Scalar = dyn_cast<llvm::yaml::ScalarNode>(&Argument);
385 if (!Scalar) {
386 ErrorMessage = "Only strings are allowed in 'arguments'.";
387 return false;
388 }
389 Command->push_back(Scalar);
390 }
391 } else if (KeyValue == "command") {
392 if (!Command)
393 Command = std::vector<llvm::yaml::ScalarNode *>(1, ValueString);
394 } else if (KeyValue == "file") {
395 File = ValueString;
396 } else if (KeyValue == "output") {
397 Output = ValueString;
398 } else {
399 ErrorMessage = ("Unknown key: \"" +
400 KeyString->getRawValue() + "\"").str();
401 return false;
402 }
403 }
404 if (!File) {
405 ErrorMessage = "Missing key: \"file\".";
406 return false;
407 }
408 if (!Command) {
409 ErrorMessage = "Missing key: \"command\" or \"arguments\".";
410 return false;
411 }
412 if (!Directory) {
413 ErrorMessage = "Missing key: \"directory\".";
414 return false;
415 }
416 SmallString<8> FileStorage;
417 StringRef FileName = File->getValue(FileStorage);
418 SmallString<128> NativeFilePath;
419 if (llvm::sys::path::is_relative(FileName)) {
420 SmallString<8> DirectoryStorage;
421 SmallString<128> AbsolutePath(
422 Directory->getValue(DirectoryStorage));
423 llvm::sys::path::append(AbsolutePath, FileName);
424 llvm::sys::path::remove_dots(AbsolutePath, /*remove_dot_dot=*/ true);
425 llvm::sys::path::native(AbsolutePath, NativeFilePath);
426 } else {
427 llvm::sys::path::native(FileName, NativeFilePath);
428 }
429 auto Cmd = CompileCommandRef(Directory, File, *Command, Output);
430 IndexByFile[NativeFilePath].push_back(Cmd);
431 AllCommands.push_back(Cmd);
432 MatchTrie.insert(NativeFilePath);
433 }
434 return true;
435}
←

/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include/llvm/Support/YAMLParser.h

1//===- YAMLParser.h - Simple YAML parser ------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is a YAML 1.2 parser.
10//
11// See http://www.yaml.org/spec/1.2/spec.html for the full standard.
12//
13// This currently does not implement the following:
14// * Multi-line literal folding.
15// * Tag resolution.
16// * UTF-16.
17// * BOMs anywhere other than the first Unicode scalar value in the file.
18//
19// The most important class here is Stream. This represents a YAML stream with
20// 0, 1, or many documents.
21//
22// SourceMgr sm;
23// StringRef input = getInput();
24// yaml::Stream stream(input, sm);
25//
26// for (yaml::document_iterator di = stream.begin(), de = stream.end();
27// di != de; ++di) {
28// yaml::Node *n = di->getRoot();
29// if (n) {
30// // Do something with n...
31// } else
32// break;
33// }
34//
35//===----------------------------------------------------------------------===//
36
37#ifndef LLVM_SUPPORT_YAMLPARSER_H
38#define LLVM_SUPPORT_YAMLPARSER_H
39
40#include "llvm/ADT/StringRef.h"
41#include "llvm/Support/Allocator.h"
42#include "llvm/Support/SMLoc.h"
43#include <cassert>
44#include <cstddef>
45#include <iterator>
46#include <map>
47#include <memory>
48#include <string>
49#include <system_error>
50
51namespace llvm {
52
53class MemoryBufferRef;
54class SourceMgr;
55class raw_ostream;
56class Twine;
57
58namespace yaml {
59
60class Document;
61class document_iterator;
62class Node;
63class Scanner;
64struct Token;
65
66/// Dump all the tokens in this stream to OS.
67/// \returns true if there was an error, false otherwise.
68bool dumpTokens(StringRef Input, raw_ostream &);
69
70/// Scans all tokens in input without outputting anything. This is used
71/// for benchmarking the tokenizer.
72/// \returns true if there was an error, false otherwise.
73bool scanTokens(StringRef Input);
74
75/// Escape \a Input for a double quoted scalar; if \p EscapePrintable
76/// is true, all UTF8 sequences will be escaped, if \p EscapePrintable is
77/// false, those UTF8 sequences encoding printable unicode scalars will not be
78/// escaped, but emitted verbatim.
79std::string escape(StringRef Input, bool EscapePrintable = true);
80
81/// This class represents a YAML stream potentially containing multiple
82/// documents.
83class Stream {
84public:
85 /// This keeps a reference to the string referenced by \p Input.
86 Stream(StringRef Input, SourceMgr &, bool ShowColors = true,
87 std::error_code *EC = nullptr);
88
89 Stream(MemoryBufferRef InputBuffer, SourceMgr &, bool ShowColors = true,
90 std::error_code *EC = nullptr);
91 ~Stream();
92
93 document_iterator begin();
94 document_iterator end();
95 void skip();
96 bool failed();
97
98 bool validate() {
99 skip();
100 return !failed();
101 }
102
103 void printError(Node *N, const Twine &Msg);
104
105private:
106 friend class Document;
107
108 std::unique_ptr<Scanner> scanner;
109 std::unique_ptr<Document> CurrentDoc;
110};
111
112/// Abstract base class for all Nodes.
113class Node {
114 virtual void anchor();
115
116public:
117 enum NodeKind {
118 NK_Null,
119 NK_Scalar,
120 NK_BlockScalar,
121 NK_KeyValue,
122 NK_Mapping,
123 NK_Sequence,
124 NK_Alias
125 };
126
127 Node(unsigned int Type, std::unique_ptr<Document> &, StringRef Anchor,
128 StringRef Tag);
129
130 // It's not safe to copy YAML nodes; the document is streamed and the position
131 // is part of the state.
132 Node(const Node &) = delete;
133 void operator=(const Node &) = delete;
134
135 void *operator new(size_t Size, BumpPtrAllocator &Alloc,
136 size_t Alignment = 16) noexcept {
137 return Alloc.Allocate(Size, Alignment);
138 }
139
140 void operator delete(void *Ptr, BumpPtrAllocator &Alloc,
141 size_t Size) noexcept {
142 Alloc.Deallocate(Ptr, Size, 0);
143 }
144
145 void operator delete(void *) noexcept = delete;
146
147 /// Get the value of the anchor attached to this node. If it does not
148 /// have one, getAnchor().size() will be 0.
149 StringRef getAnchor() const { return Anchor; }
150
151 /// Get the tag as it was written in the document. This does not
152 /// perform tag resolution.
153 StringRef getRawTag() const { return Tag; }
154
155 /// Get the verbatium tag for a given Node. This performs tag resoluton
156 /// and substitution.
157 std::string getVerbatimTag() const;
158
159 SMRange getSourceRange() const { return SourceRange; }
160 void setSourceRange(SMRange SR) { SourceRange = SR; }
161
162 // These functions forward to Document and Scanner.
163 Token &peekNext();
164 Token getNext();
165 Node *parseBlockNode();
166 BumpPtrAllocator &getAllocator();
167 void setError(const Twine &Message, Token &Location) const;
168 bool failed() const;
169
170 virtual void skip() {}
171
172 unsigned int getType() const { return TypeID; }
173
174protected:
175 std::unique_ptr<Document> &Doc;
176 SMRange SourceRange;
177
178 ~Node() = default;
179
180private:
181 unsigned int TypeID;
182 StringRef Anchor;
183 /// The tag as typed in the document.
184 StringRef Tag;
185};
186
187/// A null value.
188///
189/// Example:
190/// !!null null
191class NullNode final : public Node {
192 void anchor() override;
193
194public:
195 NullNode(std::unique_ptr<Document> &D)
196 : Node(NK_Null, D, StringRef(), StringRef()) {}
197
198 static bool classof(const Node *N) { return N->getType() == NK_Null; }
199};
200
201/// A scalar node is an opaque datum that can be presented as a
202/// series of zero or more Unicode scalar values.
203///
204/// Example:
205/// Adena
206class ScalarNode final : public Node {
207 void anchor() override;
208
209public:
210 ScalarNode(std::unique_ptr<Document> &D, StringRef Anchor, StringRef Tag,
211 StringRef Val)
212 : Node(NK_Scalar, D, Anchor, Tag), Value(Val) {
213 SMLoc Start = SMLoc::getFromPointer(Val.begin());
214 SMLoc End = SMLoc::getFromPointer(Val.end());
215 SourceRange = SMRange(Start, End);
216 }
217
218 // Return Value without any escaping or folding or other fun YAML stuff. This
219 // is the exact bytes that are contained in the file (after conversion to
220 // utf8).
221 StringRef getRawValue() const { return Value; }
222
223 /// Gets the value of this node as a StringRef.
224 ///
225 /// \param Storage is used to store the content of the returned StringRef if
226 /// it requires any modification from how it appeared in the source.
227 /// This happens with escaped characters and multi-line literals.
228 StringRef getValue(SmallVectorImpl<char> &Storage) const;
229
230 static bool classof(const Node *N) {
231 return N->getType() == NK_Scalar;
232 }
233
234private:
235 StringRef Value;
236
237 StringRef unescapeDoubleQuoted(StringRef UnquotedValue,
238 StringRef::size_type Start,
239 SmallVectorImpl<char> &Storage) const;
240};
241
242/// A block scalar node is an opaque datum that can be presented as a
243/// series of zero or more Unicode scalar values.
244///
245/// Example:
246/// |
247/// Hello
248/// World
249class BlockScalarNode final : public Node {
250 void anchor() override;
251
252public:
253 BlockScalarNode(std::unique_ptr<Document> &D, StringRef Anchor, StringRef Tag,
254 StringRef Value, StringRef RawVal)
255 : Node(NK_BlockScalar, D, Anchor, Tag), Value(Value) {
256 SMLoc Start = SMLoc::getFromPointer(RawVal.begin());
257 SMLoc End = SMLoc::getFromPointer(RawVal.end());
258 SourceRange = SMRange(Start, End);
259 }
260
261 /// Gets the value of this node as a StringRef.
262 StringRef getValue() const { return Value; }
263
264 static bool classof(const Node *N) {
265 return N->getType() == NK_BlockScalar;
266 }
267
268private:
269 StringRef Value;
270};
271
272/// A key and value pair. While not technically a Node under the YAML
273/// representation graph, it is easier to treat them this way.
274///
275/// TODO: Consider making this not a child of Node.
276///
277/// Example:
278/// Section: .text
279class KeyValueNode final : public Node {
280 void anchor() override;
281
282public:
283 KeyValueNode(std::unique_ptr<Document> &D)
284 : Node(NK_KeyValue, D, StringRef(), StringRef()) {}
285
286 /// Parse and return the key.
287 ///
288 /// This may be called multiple times.
289 ///
290 /// \returns The key, or nullptr if failed() == true.
291 Node *getKey();
292
293 /// Parse and return the value.
294 ///
295 /// This may be called multiple times.
296 ///
297 /// \returns The value, or nullptr if failed() == true.
298 Node *getValue();
299
300 void skip() override {
301 if (Node *Key = getKey()) {
302 Key->skip();
303 if (Node *Val = getValue())
304 Val->skip();
305 }
306 }
307
308 static bool classof(const Node *N) {
309 return N->getType() == NK_KeyValue;
310 }
311
312private:
313 Node *Key = nullptr;
314 Node *Value = nullptr;
315};
316
317/// This is an iterator abstraction over YAML collections shared by both
318/// sequences and maps.
319///
320/// BaseT must have a ValueT* member named CurrentEntry and a member function
321/// increment() which must set CurrentEntry to 0 to create an end iterator.
322template <class BaseT, class ValueT>
323class basic_collection_iterator
324 : public std::iterator<std::input_iterator_tag, ValueT> {
325public:
326 basic_collection_iterator() = default;
327 basic_collection_iterator(BaseT *B) : Base(B) {}
328
329 ValueT *operator->() const {
330 assert(Base && Base->CurrentEntry && "Attempted to access end iterator!")((Base && Base->CurrentEntry && "Attempted to access end iterator!"
) ? static_cast<void> (0) : __assert_fail ("Base && Base->CurrentEntry && \"Attempted to access end iterator!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include/llvm/Support/YAMLParser.h"
, 330, __PRETTY_FUNCTION__));
331 return Base->CurrentEntry;
332 }
333
334 ValueT &operator*() const {
335 assert(Base && Base->CurrentEntry &&((Base && Base->CurrentEntry && "Attempted to dereference end iterator!"
) ? static_cast<void> (0) : __assert_fail ("Base && Base->CurrentEntry && \"Attempted to dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include/llvm/Support/YAMLParser.h"
, 336, __PRETTY_FUNCTION__))
336 "Attempted to dereference end iterator!")((Base && Base->CurrentEntry && "Attempted to dereference end iterator!"
) ? static_cast<void> (0) : __assert_fail ("Base && Base->CurrentEntry && \"Attempted to dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include/llvm/Support/YAMLParser.h"
, 336, __PRETTY_FUNCTION__));
337 return *Base->CurrentEntry;
338 }
339
340 operator ValueT *() const {
341 assert(Base && Base->CurrentEntry && "Attempted to access end iterator!")((Base && Base->CurrentEntry && "Attempted to access end iterator!"
) ? static_cast<void> (0) : __assert_fail ("Base && Base->CurrentEntry && \"Attempted to access end iterator!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include/llvm/Support/YAMLParser.h"
, 341, __PRETTY_FUNCTION__));
342 return Base->CurrentEntry;
343 }
344
345 /// Note on EqualityComparable:
346 ///
347 /// The iterator is not re-entrant,
348 /// it is meant to be used for parsing YAML on-demand
349 /// Once iteration started - it can point only to one entry at a time
350 /// hence Base.CurrentEntry and Other.Base.CurrentEntry are equal
351 /// iff Base and Other.Base are equal.
352 bool operator==(const basic_collection_iterator &Other) const {
353 if (Base && (Base == Other.Base)) {
354 assert((Base->CurrentEntry == Other.Base->CurrentEntry)(((Base->CurrentEntry == Other.Base->CurrentEntry) &&
"Equal Bases expected to point to equal Entries") ? static_cast
<void> (0) : __assert_fail ("(Base->CurrentEntry == Other.Base->CurrentEntry) && \"Equal Bases expected to point to equal Entries\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include/llvm/Support/YAMLParser.h"
, 355, __PRETTY_FUNCTION__))
355 && "Equal Bases expected to point to equal Entries")(((Base->CurrentEntry == Other.Base->CurrentEntry) &&
"Equal Bases expected to point to equal Entries") ? static_cast
<void> (0) : __assert_fail ("(Base->CurrentEntry == Other.Base->CurrentEntry) && \"Equal Bases expected to point to equal Entries\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include/llvm/Support/YAMLParser.h"
, 355, __PRETTY_FUNCTION__));
356 }
357
358 return Base == Other.Base;
359 }
360
361 bool operator!=(const basic_collection_iterator &Other) const {
362 return !(Base == Other.Base);
363 }
364
365 basic_collection_iterator &operator++() {
366 assert(Base && "Attempted to advance iterator past end!")((Base && "Attempted to advance iterator past end!") ?
static_cast<void> (0) : __assert_fail ("Base && \"Attempted to advance iterator past end!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include/llvm/Support/YAMLParser.h"
, 366, __PRETTY_FUNCTION__));
367 Base->increment();
368 // Create an end iterator.
369 if (!Base->CurrentEntry)
370 Base = nullptr;
371 return *this;
372 }
373
374private:
375 BaseT *Base = nullptr;
376};
377
378// The following two templates are used for both MappingNode and Sequence Node.
379template <class CollectionType>
380typename CollectionType::iterator begin(CollectionType &C) {
381 assert(C.IsAtBeginning && "You may only iterate over a collection once!")((C.IsAtBeginning && "You may only iterate over a collection once!"
) ? static_cast<void> (0) : __assert_fail ("C.IsAtBeginning && \"You may only iterate over a collection once!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include/llvm/Support/YAMLParser.h"
, 381, __PRETTY_FUNCTION__));
382 C.IsAtBeginning = false;
383 typename CollectionType::iterator ret(&C);
384 ++ret;
385 return ret;
386}
387
388template <class CollectionType> void skip(CollectionType &C) {
389 // TODO: support skipping from the middle of a parsed collection ;/
390 assert((C.IsAtBeginning || C.IsAtEnd) && "Cannot skip mid parse!")(((C.IsAtBeginning || C.IsAtEnd) && "Cannot skip mid parse!"
) ? static_cast<void> (0) : __assert_fail ("(C.IsAtBeginning || C.IsAtEnd) && \"Cannot skip mid parse!\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include/llvm/Support/YAMLParser.h"
, 390, __PRETTY_FUNCTION__));
391 if (C.IsAtBeginning)
392 for (typename CollectionType::iterator i = begin(C), e = C.end(); i != e;
393 ++i)
394 i->skip();
395}
396
397/// Represents a YAML map created from either a block map for a flow map.
398///
399/// This parses the YAML stream as increment() is called.
400///
401/// Example:
402/// Name: _main
403/// Scope: Global
404class MappingNode final : public Node {
405 void anchor() override;
406
407public:
408 enum MappingType {
409 MT_Block,
410 MT_Flow,
411 MT_Inline ///< An inline mapping node is used for "[key: value]".
412 };
413
414 MappingNode(std::unique_ptr<Document> &D, StringRef Anchor, StringRef Tag,
415 MappingType MT)
416 : Node(NK_Mapping, D, Anchor, Tag), Type(MT) {}
417
418 friend class basic_collection_iterator<MappingNode, KeyValueNode>;
419
420 using iterator = basic_collection_iterator<MappingNode, KeyValueNode>;
421
422 template <class T> friend typename T::iterator yaml::begin(T &);
423 template <class T> friend void yaml::skip(T &);
424
425 iterator begin() { return yaml::begin(*this); }
426
427 iterator end() { return iterator(); }
428
429 void skip() override { yaml::skip(*this); }
430
431 static bool classof(const Node *N) {
432 return N->getType() == NK_Mapping;
433 }
434
435private:
436 MappingType Type;
437 bool IsAtBeginning = true;
438 bool IsAtEnd = false;
439 KeyValueNode *CurrentEntry = nullptr;
440
441 void increment();
442};
443
444/// Represents a YAML sequence created from either a block sequence for a
445/// flow sequence.
446///
447/// This parses the YAML stream as increment() is called.
448///
449/// Example:
450/// - Hello
451/// - World
452class SequenceNode final : public Node {
453 void anchor() override;
454
455public:
456 enum SequenceType {
457 ST_Block,
458 ST_Flow,
459 // Use for:
460 //
461 // key:
462 // - val1
463 // - val2
464 //
465 // As a BlockMappingEntry and BlockEnd are not created in this case.
466 ST_Indentless
467 };
468
469 SequenceNode(std::unique_ptr<Document> &D, StringRef Anchor, StringRef Tag,
470 SequenceType ST)
471 : Node(NK_Sequence, D, Anchor, Tag), SeqType(ST) {}
472
473 friend class basic_collection_iterator<SequenceNode, Node>;
474
475 using iterator = basic_collection_iterator<SequenceNode, Node>;
476
477 template <class T> friend typename T::iterator yaml::begin(T &);
478 template <class T> friend void yaml::skip(T &);
479
480 void increment();
481
482 iterator begin() { return yaml::begin(*this); }
483
484 iterator end() { return iterator(); }
485
486 void skip() override { yaml::skip(*this); }
487
488 static bool classof(const Node *N) {
489 return N->getType() == NK_Sequence;
490 }
491
492private:
493 SequenceType SeqType;
494 bool IsAtBeginning = true;
495 bool IsAtEnd = false;
496 bool WasPreviousTokenFlowEntry = true; // Start with an imaginary ','.
497 Node *CurrentEntry = nullptr;
498};
499
500/// Represents an alias to a Node with an anchor.
501///
502/// Example:
503/// *AnchorName
504class AliasNode final : public Node {
505 void anchor() override;
506
507public:
508 AliasNode(std::unique_ptr<Document> &D, StringRef Val)
509 : Node(NK_Alias, D, StringRef(), StringRef()), Name(Val) {}
510
511 StringRef getName() const { return Name; }
512 Node *getTarget();
513
514 static bool classof(const Node *N) { return N->getType() == NK_Alias; }
515
516private:
517 StringRef Name;
518};
519
520/// A YAML Stream is a sequence of Documents. A document contains a root
521/// node.
522class Document {
523public:
524 Document(Stream &ParentStream);
525
526 /// Root for parsing a node. Returns a single node.
527 Node *parseBlockNode();
528
529 /// Finish parsing the current document and return true if there are
530 /// more. Return false otherwise.
531 bool skip();
532
533 /// Parse and return the root level node.
534 Node *getRoot() {
535 if (Root)
17
←
Assuming field 'Root' is non-null, which participates in a condition later
→
18
←
Taking true branch
→
536 return Root;
19
←
Returning pointer, which participates in a condition later
→
537 return Root = parseBlockNode();
538 }
539
540 const std::map<StringRef, StringRef> &getTagMap() const { return TagMap; }
541
542private:
543 friend class Node;
544 friend class document_iterator;
545
546 /// Stream to read tokens from.
547 Stream &stream;
548
549 /// Used to allocate nodes to. All are destroyed without calling their
550 /// destructor when the document is destroyed.
551 BumpPtrAllocator NodeAllocator;
552
553 /// The root node. Used to support skipping a partially parsed
554 /// document.
555 Node *Root;
556
557 /// Maps tag prefixes to their expansion.
558 std::map<StringRef, StringRef> TagMap;
559
560 Token &peekNext();
561 Token getNext();
562 void setError(const Twine &Message, Token &Location) const;
563 bool failed() const;
564
565 /// Parse %BLAH directives and return true if any were encountered.
566 bool parseDirectives();
567
568 /// Parse %YAML
569 void parseYAMLDirective();
570
571 /// Parse %TAG
572 void parseTAGDirective();
573
574 /// Consume the next token and error if it is not \a TK.
575 bool expectToken(int TK);
576};
577
578/// Iterator abstraction for Documents over a Stream.
579class document_iterator {
580public:
581 document_iterator() = default;
582 document_iterator(std::unique_ptr<Document> &D) : Doc(&D) {}
583
584 bool operator==(const document_iterator &Other) const {
585 if (isAtEnd() || Other.isAtEnd())
3
←
Calling 'document_iterator::isAtEnd'
→
6
←
Returning from 'document_iterator::isAtEnd'
→
7
←
Calling 'document_iterator::isAtEnd'
→
10
←
Returning from 'document_iterator::isAtEnd'
→
11
←
Taking false branch
→
586 return isAtEnd() && Other.isAtEnd();
587
588 return Doc == Other.Doc;
12
←
Assuming 'Doc' is not equal to 'Other.Doc'
→
13
←
Returning zero, which participates in a condition later
→
589 }
590 bool operator!=(const document_iterator &Other) const {
591 return !(*this == Other);
592 }
593
594 document_iterator operator++() {
595 assert(Doc && "incrementing iterator past the end.")((Doc && "incrementing iterator past the end.") ? static_cast
<void> (0) : __assert_fail ("Doc && \"incrementing iterator past the end.\""
, "/build/llvm-toolchain-snapshot-12~++20200917111122+b03c2b8395b/llvm/include/llvm/Support/YAMLParser.h"
, 595, __PRETTY_FUNCTION__));
596 if (!(*Doc)->skip()) {
597 Doc->reset(nullptr);
598 } else {
599 Stream &S = (*Doc)->stream;
600 Doc->reset(new Document(S));
601 }
602 return *this;
603 }
604
605 Document &operator*() { return *Doc->get(); }
606
607 std::unique_ptr<Document> &operator->() { return *Doc; }
608
609private:
610 bool isAtEnd() const { return !Doc || !*Doc; }
4
←
Assuming field 'Doc' is non-null
→
5
←
Returning zero, which participates in a condition later
→
8
←
Assuming field 'Doc' is non-null
→
9
←
Returning zero, which participates in a condition later
→
611
612 std::unique_ptr<Document> *Doc = nullptr;
613};
614
615} // end namespace yaml
616
617} // end namespace llvm
618
619#endif // LLVM_SUPPORT_YAMLPARSER_H