Bug Summary

File:clang/lib/Tooling/JSONCompilationDatabase.cpp
Warning:line 381, column 31
Dereference of null pointer (loaded from variable 'SequenceString')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name JSONCompilationDatabase.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -relaxed-aliasing -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D CLANG_VENDOR="Debian " -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/tools/clang/lib/Tooling -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/Tooling -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/tools/clang/lib/Tooling -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-12-07-102640-14763-1 -x c++ /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/Tooling/JSONCompilationDatabase.cpp

/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/Tooling/JSONCompilationDatabase.cpp

1//===- JSONCompilationDatabase.cpp ----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the implementation of the JSONCompilationDatabase.
10//
11//===----------------------------------------------------------------------===//
12
13#include "clang/Tooling/JSONCompilationDatabase.h"
14#include "clang/Basic/LLVM.h"
15#include "clang/Tooling/CompilationDatabase.h"
16#include "clang/Tooling/CompilationDatabasePluginRegistry.h"
17#include "clang/Tooling/Tooling.h"
18#include "llvm/ADT/Optional.h"
19#include "llvm/ADT/STLExtras.h"
20#include "llvm/ADT/SmallString.h"
21#include "llvm/ADT/SmallVector.h"
22#include "llvm/ADT/StringRef.h"
23#include "llvm/ADT/Triple.h"
24#include "llvm/Support/Allocator.h"
25#include "llvm/Support/Casting.h"
26#include "llvm/Support/CommandLine.h"
27#include "llvm/Support/ErrorOr.h"
28#include "llvm/Support/Host.h"
29#include "llvm/Support/MemoryBuffer.h"
30#include "llvm/Support/Path.h"
31#include "llvm/Support/StringSaver.h"
32#include "llvm/Support/YAMLParser.h"
33#include "llvm/Support/raw_ostream.h"
34#include <cassert>
35#include <memory>
36#include <string>
37#include <system_error>
38#include <tuple>
39#include <utility>
40#include <vector>
41
42using namespace clang;
43using namespace tooling;
44
45namespace {
46
47/// A parser for escaped strings of command line arguments.
48///
49/// Assumes \-escaping for quoted arguments (see the documentation of
50/// unescapeCommandLine(...)).
51class CommandLineArgumentParser {
52 public:
53 CommandLineArgumentParser(StringRef CommandLine)
54 : Input(CommandLine), Position(Input.begin()-1) {}
55
56 std::vector<std::string> parse() {
57 bool HasMoreInput = true;
58 while (HasMoreInput && nextNonWhitespace()) {
59 std::string Argument;
60 HasMoreInput = parseStringInto(Argument);
61 CommandLine.push_back(Argument);
62 }
63 return CommandLine;
64 }
65
66 private:
67 // All private methods return true if there is more input available.
68
69 bool parseStringInto(std::string &String) {
70 do {
71 if (*Position == '"') {
72 if (!parseDoubleQuotedStringInto(String)) return false;
73 } else if (*Position == '\'') {
74 if (!parseSingleQuotedStringInto(String)) return false;
75 } else {
76 if (!parseFreeStringInto(String)) return false;
77 }
78 } while (*Position != ' ');
79 return true;
80 }
81
82 bool parseDoubleQuotedStringInto(std::string &String) {
83 if (!next()) return false;
84 while (*Position != '"') {
85 if (!skipEscapeCharacter()) return false;
86 String.push_back(*Position);
87 if (!next()) return false;
88 }
89 return next();
90 }
91
92 bool parseSingleQuotedStringInto(std::string &String) {
93 if (!next()) return false;
94 while (*Position != '\'') {
95 String.push_back(*Position);
96 if (!next()) return false;
97 }
98 return next();
99 }
100
101 bool parseFreeStringInto(std::string &String) {
102 do {
103 if (!skipEscapeCharacter()) return false;
104 String.push_back(*Position);
105 if (!next()) return false;
106 } while (*Position != ' ' && *Position != '"' && *Position != '\'');
107 return true;
108 }
109
110 bool skipEscapeCharacter() {
111 if (*Position == '\\') {
112 return next();
113 }
114 return true;
115 }
116
117 bool nextNonWhitespace() {
118 do {
119 if (!next()) return false;
120 } while (*Position == ' ');
121 return true;
122 }
123
124 bool next() {
125 ++Position;
126 return Position != Input.end();
127 }
128
129 const StringRef Input;
130 StringRef::iterator Position;
131 std::vector<std::string> CommandLine;
132};
133
134std::vector<std::string> unescapeCommandLine(JSONCommandLineSyntax Syntax,
135 StringRef EscapedCommandLine) {
136 if (Syntax == JSONCommandLineSyntax::AutoDetect) {
137 Syntax = JSONCommandLineSyntax::Gnu;
138 llvm::Triple Triple(llvm::sys::getProcessTriple());
139 if (Triple.getOS() == llvm::Triple::OSType::Win32) {
140 // Assume Windows command line parsing on Win32 unless the triple
141 // explicitly tells us otherwise.
142 if (!Triple.hasEnvironment() ||
143 Triple.getEnvironment() == llvm::Triple::EnvironmentType::MSVC)
144 Syntax = JSONCommandLineSyntax::Windows;
145 }
146 }
147
148 if (Syntax == JSONCommandLineSyntax::Windows) {
149 llvm::BumpPtrAllocator Alloc;
150 llvm::StringSaver Saver(Alloc);
151 llvm::SmallVector<const char *, 64> T;
152 llvm::cl::TokenizeWindowsCommandLine(EscapedCommandLine, Saver, T);
153 std::vector<std::string> Result(T.begin(), T.end());
154 return Result;
155 }
156 assert(Syntax == JSONCommandLineSyntax::Gnu)((Syntax == JSONCommandLineSyntax::Gnu) ? static_cast<void
> (0) : __assert_fail ("Syntax == JSONCommandLineSyntax::Gnu"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/clang/lib/Tooling/JSONCompilationDatabase.cpp"
, 156, __PRETTY_FUNCTION__))
;
157 CommandLineArgumentParser parser(EscapedCommandLine);
158 return parser.parse();
159}
160
161// This plugin locates a nearby compile_command.json file, and also infers
162// compile commands for files not present in the database.
163class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
164 std::unique_ptr<CompilationDatabase>
165 loadFromDirectory(StringRef Directory, std::string &ErrorMessage) override {
166 SmallString<1024> JSONDatabasePath(Directory);
167 llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
168 auto Base = JSONCompilationDatabase::loadFromFile(
169 JSONDatabasePath, ErrorMessage, JSONCommandLineSyntax::AutoDetect);
170 return Base ? inferTargetAndDriverMode(
171 inferMissingCompileCommands(std::move(Base)))
172 : nullptr;
173 }
174};
175
176} // namespace
177
178// Register the JSONCompilationDatabasePlugin with the
179// CompilationDatabasePluginRegistry using this statically initialized variable.
180static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>
181X("json-compilation-database", "Reads JSON formatted compilation databases");
182
183namespace clang {
184namespace tooling {
185
186// This anchor is used to force the linker to link in the generated object file
187// and thus register the JSONCompilationDatabasePlugin.
188volatile int JSONAnchorSource = 0;
189
190} // namespace tooling
191} // namespace clang
192
193std::unique_ptr<JSONCompilationDatabase>
194JSONCompilationDatabase::loadFromFile(StringRef FilePath,
195 std::string &ErrorMessage,
196 JSONCommandLineSyntax Syntax) {
197 // Don't mmap: if we're a long-lived process, the build system may overwrite.
198 llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> DatabaseBuffer =
199 llvm::MemoryBuffer::getFile(FilePath, /*FileSize=*/-1,
200 /*RequiresNullTerminator=*/true,
201 /*IsVolatile=*/true);
202 if (std::error_code Result = DatabaseBuffer.getError()) {
203 ErrorMessage = "Error while opening JSON database: " + Result.message();
204 return nullptr;
205 }
206 std::unique_ptr<JSONCompilationDatabase> Database(
207 new JSONCompilationDatabase(std::move(*DatabaseBuffer), Syntax));
208 if (!Database->parse(ErrorMessage))
209 return nullptr;
210 return Database;
211}
212
213std::unique_ptr<JSONCompilationDatabase>
214JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString,
215 std::string &ErrorMessage,
216 JSONCommandLineSyntax Syntax) {
217 std::unique_ptr<llvm::MemoryBuffer> DatabaseBuffer(
218 llvm::MemoryBuffer::getMemBuffer(DatabaseString));
219 std::unique_ptr<JSONCompilationDatabase> Database(
220 new JSONCompilationDatabase(std::move(DatabaseBuffer), Syntax));
221 if (!Database->parse(ErrorMessage))
1
Calling 'JSONCompilationDatabase::parse'
222 return nullptr;
223 return Database;
224}
225
226std::vector<CompileCommand>
227JSONCompilationDatabase::getCompileCommands(StringRef FilePath) const {
228 SmallString<128> NativeFilePath;
229 llvm::sys::path::native(FilePath, NativeFilePath);
230
231 std::string Error;
232 llvm::raw_string_ostream ES(Error);
233 StringRef Match = MatchTrie.findEquivalent(NativeFilePath, ES);
234 if (Match.empty())
235 return {};
236 const auto CommandsRefI = IndexByFile.find(Match);
237 if (CommandsRefI == IndexByFile.end())
238 return {};
239 std::vector<CompileCommand> Commands;
240 getCommands(CommandsRefI->getValue(), Commands);
241 return Commands;
242}
243
244std::vector<std::string>
245JSONCompilationDatabase::getAllFiles() const {
246 std::vector<std::string> Result;
247 for (const auto &CommandRef : IndexByFile)
248 Result.push_back(CommandRef.first().str());
249 return Result;
250}
251
252std::vector<CompileCommand>
253JSONCompilationDatabase::getAllCompileCommands() const {
254 std::vector<CompileCommand> Commands;
255 getCommands(AllCommands, Commands);
256 return Commands;
257}
258
259static llvm::StringRef stripExecutableExtension(llvm::StringRef Name) {
260 Name.consume_back(".exe");
261 return Name;
262}
263
264// There are compiler-wrappers (ccache, distcc, gomacc) that take the "real"
265// compiler as an argument, e.g. distcc gcc -O3 foo.c.
266// These end up in compile_commands.json when people set CC="distcc gcc".
267// Clang's driver doesn't understand this, so we need to unwrap.
268static bool unwrapCommand(std::vector<std::string> &Args) {
269 if (Args.size() < 2)
270 return false;
271 StringRef Wrapper =
272 stripExecutableExtension(llvm::sys::path::filename(Args.front()));
273 if (Wrapper == "distcc" || Wrapper == "gomacc" || Wrapper == "ccache") {
274 // Most of these wrappers support being invoked 3 ways:
275 // `distcc g++ file.c` This is the mode we're trying to match.
276 // We need to drop `distcc`.
277 // `distcc file.c` This acts like compiler is cc or similar.
278 // Clang's driver can handle this, no change needed.
279 // `g++ file.c` g++ is a symlink to distcc.
280 // We don't even notice this case, and all is well.
281 //
282 // We need to distinguish between the first and second case.
283 // The wrappers themselves don't take flags, so Args[1] is a compiler flag,
284 // an input file, or a compiler. Inputs have extensions, compilers don't.
285 bool HasCompiler =
286 (Args[1][0] != '-') &&
287 !llvm::sys::path::has_extension(stripExecutableExtension(Args[1]));
288 if (HasCompiler) {
289 Args.erase(Args.begin());
290 return true;
291 }
292 // If !HasCompiler, wrappers act like GCC. Fine: so do we.
293 }
294 return false;
295}
296
297static std::vector<std::string>
298nodeToCommandLine(JSONCommandLineSyntax Syntax,
299 const std::vector<llvm::yaml::ScalarNode *> &Nodes) {
300 SmallString<1024> Storage;
301 std::vector<std::string> Arguments;
302 if (Nodes.size() == 1)
303 Arguments = unescapeCommandLine(Syntax, Nodes[0]->getValue(Storage));
304 else
305 for (const auto *Node : Nodes)
306 Arguments.push_back(Node->getValue(Storage));
307 // There may be multiple wrappers: using distcc and ccache together is common.
308 while (unwrapCommand(Arguments))
309 ;
310 return Arguments;
311}
312
313void JSONCompilationDatabase::getCommands(
314 ArrayRef<CompileCommandRef> CommandsRef,
315 std::vector<CompileCommand> &Commands) const {
316 for (const auto &CommandRef : CommandsRef) {
317 SmallString<8> DirectoryStorage;
318 SmallString<32> FilenameStorage;
319 SmallString<32> OutputStorage;
320 auto Output = std::get<3>(CommandRef);
321 Commands.emplace_back(
322 std::get<0>(CommandRef)->getValue(DirectoryStorage),
323 std::get<1>(CommandRef)->getValue(FilenameStorage),
324 nodeToCommandLine(Syntax, std::get<2>(CommandRef)),
325 Output ? Output->getValue(OutputStorage) : "");
326 }
327}
328
329bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
330 llvm::yaml::document_iterator I = YAMLStream.begin();
331 if (I == YAMLStream.end()) {
2
Calling 'document_iterator::operator=='
14
Returning from 'document_iterator::operator=='
15
Taking false branch
332 ErrorMessage = "Error while parsing YAML.";
333 return false;
334 }
335 llvm::yaml::Node *Root = I->getRoot();
16
Calling 'Document::getRoot'
20
Returning from 'Document::getRoot'
336 if (!Root
20.1
'Root' is non-null
20.1
'Root' is non-null
) {
21
Taking false branch
337 ErrorMessage = "Error while parsing YAML.";
338 return false;
339 }
340 auto *Array = dyn_cast<llvm::yaml::SequenceNode>(Root);
22
Assuming 'Root' is a 'SequenceNode'
341 if (!Array
22.1
'Array' is non-null
22.1
'Array' is non-null
) {
23
Taking false branch
342 ErrorMessage = "Expected array.";
343 return false;
344 }
345 for (auto &NextObject : *Array) {
346 auto *Object = dyn_cast<llvm::yaml::MappingNode>(&NextObject);
24
Assuming the object is a 'MappingNode'
347 if (!Object
24.1
'Object' is non-null
24.1
'Object' is non-null
) {
25
Taking false branch
348 ErrorMessage = "Expected object.";
349 return false;
350 }
351 llvm::yaml::ScalarNode *Directory = nullptr;
352 llvm::Optional<std::vector<llvm::yaml::ScalarNode *>> Command;
353 llvm::yaml::ScalarNode *File = nullptr;
354 llvm::yaml::ScalarNode *Output = nullptr;
355 for (auto& NextKeyValue : *Object) {
356 auto *KeyString = dyn_cast<llvm::yaml::ScalarNode>(NextKeyValue.getKey());
26
Assuming the object is a 'ScalarNode'
357 if (!KeyString
26.1
'KeyString' is non-null
26.1
'KeyString' is non-null
) {
27
Taking false branch
358 ErrorMessage = "Expected strings as key.";
359 return false;
360 }
361 SmallString<10> KeyStorage;
362 StringRef KeyValue = KeyString->getValue(KeyStorage);
363 llvm::yaml::Node *Value = NextKeyValue.getValue();
364 if (!Value) {
28
Assuming 'Value' is non-null
29
Taking false branch
365 ErrorMessage = "Expected value.";
366 return false;
367 }
368 auto *ValueString = dyn_cast<llvm::yaml::ScalarNode>(Value);
30
Assuming 'Value' is not a 'ScalarNode'
369 auto *SequenceString = dyn_cast<llvm::yaml::SequenceNode>(Value);
31
Assuming 'Value' is not a 'SequenceNode'
32
'SequenceString' initialized to a null pointer value
370 if (KeyValue == "arguments" && !SequenceString) {
33
Assuming the condition is false
34
Taking false branch
371 ErrorMessage = "Expected sequence as value.";
372 return false;
373 } else if (KeyValue != "arguments" && !ValueString) {
35
Taking false branch
374 ErrorMessage = "Expected string as value.";
375 return false;
376 }
377 if (KeyValue == "directory") {
36
Assuming the condition is false
37
Taking false branch
378 Directory = ValueString;
379 } else if (KeyValue == "arguments") {
38
Assuming the condition is true
39
Taking true branch
380 Command = std::vector<llvm::yaml::ScalarNode *>();
381 for (auto &Argument : *SequenceString) {
40
Dereference of null pointer (loaded from variable 'SequenceString')
382 auto *Scalar = dyn_cast<llvm::yaml::ScalarNode>(&Argument);
383 if (!Scalar) {
384 ErrorMessage = "Only strings are allowed in 'arguments'.";
385 return false;
386 }
387 Command->push_back(Scalar);
388 }
389 } else if (KeyValue == "command") {
390 if (!Command)
391 Command = std::vector<llvm::yaml::ScalarNode *>(1, ValueString);
392 } else if (KeyValue == "file") {
393 File = ValueString;
394 } else if (KeyValue == "output") {
395 Output = ValueString;
396 } else {
397 ErrorMessage = ("Unknown key: \"" +
398 KeyString->getRawValue() + "\"").str();
399 return false;
400 }
401 }
402 if (!File) {
403 ErrorMessage = "Missing key: \"file\".";
404 return false;
405 }
406 if (!Command) {
407 ErrorMessage = "Missing key: \"command\" or \"arguments\".";
408 return false;
409 }
410 if (!Directory) {
411 ErrorMessage = "Missing key: \"directory\".";
412 return false;
413 }
414 SmallString<8> FileStorage;
415 StringRef FileName = File->getValue(FileStorage);
416 SmallString<128> NativeFilePath;
417 if (llvm::sys::path::is_relative(FileName)) {
418 SmallString<8> DirectoryStorage;
419 SmallString<128> AbsolutePath(
420 Directory->getValue(DirectoryStorage));
421 llvm::sys::path::append(AbsolutePath, FileName);
422 llvm::sys::path::remove_dots(AbsolutePath, /*remove_dot_dot=*/ true);
423 llvm::sys::path::native(AbsolutePath, NativeFilePath);
424 } else {
425 llvm::sys::path::native(FileName, NativeFilePath);
426 }
427 auto Cmd = CompileCommandRef(Directory, File, *Command, Output);
428 IndexByFile[NativeFilePath].push_back(Cmd);
429 AllCommands.push_back(Cmd);
430 MatchTrie.insert(NativeFilePath);
431 }
432 return true;
433}

/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/Support/YAMLParser.h

1//===- YAMLParser.h - Simple YAML parser ------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This is a YAML 1.2 parser.
10//
11// See http://www.yaml.org/spec/1.2/spec.html for the full standard.
12//
13// This currently does not implement the following:
14// * Multi-line literal folding.
15// * Tag resolution.
16// * UTF-16.
17// * BOMs anywhere other than the first Unicode scalar value in the file.
18//
19// The most important class here is Stream. This represents a YAML stream with
20// 0, 1, or many documents.
21//
22// SourceMgr sm;
23// StringRef input = getInput();
24// yaml::Stream stream(input, sm);
25//
26// for (yaml::document_iterator di = stream.begin(), de = stream.end();
27// di != de; ++di) {
28// yaml::Node *n = di->getRoot();
29// if (n) {
30// // Do something with n...
31// } else
32// break;
33// }
34//
35//===----------------------------------------------------------------------===//
36
37#ifndef LLVM_SUPPORT_YAMLPARSER_H
38#define LLVM_SUPPORT_YAMLPARSER_H
39
40#include "llvm/ADT/StringRef.h"
41#include "llvm/Support/Allocator.h"
42#include "llvm/Support/SMLoc.h"
43#include <cassert>
44#include <cstddef>
45#include <iterator>
46#include <map>
47#include <memory>
48#include <string>
49#include <system_error>
50
51namespace llvm {
52
53class MemoryBufferRef;
54class SourceMgr;
55class raw_ostream;
56class Twine;
57
58namespace yaml {
59
60class Document;
61class document_iterator;
62class Node;
63class Scanner;
64struct Token;
65
66/// Dump all the tokens in this stream to OS.
67/// \returns true if there was an error, false otherwise.
68bool dumpTokens(StringRef Input, raw_ostream &);
69
70/// Scans all tokens in input without outputting anything. This is used
71/// for benchmarking the tokenizer.
72/// \returns true if there was an error, false otherwise.
73bool scanTokens(StringRef Input);
74
75/// Escape \a Input for a double quoted scalar; if \p EscapePrintable
76/// is true, all UTF8 sequences will be escaped, if \p EscapePrintable is
77/// false, those UTF8 sequences encoding printable unicode scalars will not be
78/// escaped, but emitted verbatim.
79std::string escape(StringRef Input, bool EscapePrintable = true);
80
81/// This class represents a YAML stream potentially containing multiple
82/// documents.
83class Stream {
84public:
85 /// This keeps a reference to the string referenced by \p Input.
86 Stream(StringRef Input, SourceMgr &, bool ShowColors = true,
87 std::error_code *EC = nullptr);
88
89 Stream(MemoryBufferRef InputBuffer, SourceMgr &, bool ShowColors = true,
90 std::error_code *EC = nullptr);
91 ~Stream();
92
93 document_iterator begin();
94 document_iterator end();
95 void skip();
96 bool failed();
97
98 bool validate() {
99 skip();
100 return !failed();
101 }
102
103 void printError(Node *N, const Twine &Msg);
104
105private:
106 friend class Document;
107
108 std::unique_ptr<Scanner> scanner;
109 std::unique_ptr<Document> CurrentDoc;
110};
111
112/// Abstract base class for all Nodes.
113class Node {
114 virtual void anchor();
115
116public:
117 enum NodeKind {
118 NK_Null,
119 NK_Scalar,
120 NK_BlockScalar,
121 NK_KeyValue,
122 NK_Mapping,
123 NK_Sequence,
124 NK_Alias
125 };
126
127 Node(unsigned int Type, std::unique_ptr<Document> &, StringRef Anchor,
128 StringRef Tag);
129
130 // It's not safe to copy YAML nodes; the document is streamed and the position
131 // is part of the state.
132 Node(const Node &) = delete;
133 void operator=(const Node &) = delete;
134
135 void *operator new(size_t Size, BumpPtrAllocator &Alloc,
136 size_t Alignment = 16) noexcept {
137 return Alloc.Allocate(Size, Alignment);
138 }
139
140 void operator delete(void *Ptr, BumpPtrAllocator &Alloc,
141 size_t Size) noexcept {
142 Alloc.Deallocate(Ptr, Size);
143 }
144
145 void operator delete(void *) noexcept = delete;
146
147 /// Get the value of the anchor attached to this node. If it does not
148 /// have one, getAnchor().size() will be 0.
149 StringRef getAnchor() const { return Anchor; }
150
151 /// Get the tag as it was written in the document. This does not
152 /// perform tag resolution.
153 StringRef getRawTag() const { return Tag; }
154
155 /// Get the verbatium tag for a given Node. This performs tag resoluton
156 /// and substitution.
157 std::string getVerbatimTag() const;
158
159 SMRange getSourceRange() const { return SourceRange; }
160 void setSourceRange(SMRange SR) { SourceRange = SR; }
161
162 // These functions forward to Document and Scanner.
163 Token &peekNext();
164 Token getNext();
165 Node *parseBlockNode();
166 BumpPtrAllocator &getAllocator();
167 void setError(const Twine &Message, Token &Location) const;
168 bool failed() const;
169
170 virtual void skip() {}
171
172 unsigned int getType() const { return TypeID; }
173
174protected:
175 std::unique_ptr<Document> &Doc;
176 SMRange SourceRange;
177
178 ~Node() = default;
179
180private:
181 unsigned int TypeID;
182 StringRef Anchor;
183 /// The tag as typed in the document.
184 StringRef Tag;
185};
186
187/// A null value.
188///
189/// Example:
190/// !!null null
191class NullNode final : public Node {
192 void anchor() override;
193
194public:
195 NullNode(std::unique_ptr<Document> &D)
196 : Node(NK_Null, D, StringRef(), StringRef()) {}
197
198 static bool classof(const Node *N) { return N->getType() == NK_Null; }
199};
200
201/// A scalar node is an opaque datum that can be presented as a
202/// series of zero or more Unicode scalar values.
203///
204/// Example:
205/// Adena
206class ScalarNode final : public Node {
207 void anchor() override;
208
209public:
210 ScalarNode(std::unique_ptr<Document> &D, StringRef Anchor, StringRef Tag,
211 StringRef Val)
212 : Node(NK_Scalar, D, Anchor, Tag), Value(Val) {
213 SMLoc Start = SMLoc::getFromPointer(Val.begin());
214 SMLoc End = SMLoc::getFromPointer(Val.end());
215 SourceRange = SMRange(Start, End);
216 }
217
218 // Return Value without any escaping or folding or other fun YAML stuff. This
219 // is the exact bytes that are contained in the file (after conversion to
220 // utf8).
221 StringRef getRawValue() const { return Value; }
222
223 /// Gets the value of this node as a StringRef.
224 ///
225 /// \param Storage is used to store the content of the returned StringRef iff
226 /// it requires any modification from how it appeared in the source.
227 /// This happens with escaped characters and multi-line literals.
228 StringRef getValue(SmallVectorImpl<char> &Storage) const;
229
230 static bool classof(const Node *N) {
231 return N->getType() == NK_Scalar;
232 }
233
234private:
235 StringRef Value;
236
237 StringRef unescapeDoubleQuoted(StringRef UnquotedValue,
238 StringRef::size_type Start,
239 SmallVectorImpl<char> &Storage) const;
240};
241
242/// A block scalar node is an opaque datum that can be presented as a
243/// series of zero or more Unicode scalar values.
244///
245/// Example:
246/// |
247/// Hello
248/// World
249class BlockScalarNode final : public Node {
250 void anchor() override;
251
252public:
253 BlockScalarNode(std::unique_ptr<Document> &D, StringRef Anchor, StringRef Tag,
254 StringRef Value, StringRef RawVal)
255 : Node(NK_BlockScalar, D, Anchor, Tag), Value(Value) {
256 SMLoc Start = SMLoc::getFromPointer(RawVal.begin());
257 SMLoc End = SMLoc::getFromPointer(RawVal.end());
258 SourceRange = SMRange(Start, End);
259 }
260
261 /// Gets the value of this node as a StringRef.
262 StringRef getValue() const { return Value; }
263
264 static bool classof(const Node *N) {
265 return N->getType() == NK_BlockScalar;
266 }
267
268private:
269 StringRef Value;
270};
271
272/// A key and value pair. While not technically a Node under the YAML
273/// representation graph, it is easier to treat them this way.
274///
275/// TODO: Consider making this not a child of Node.
276///
277/// Example:
278/// Section: .text
279class KeyValueNode final : public Node {
280 void anchor() override;
281
282public:
283 KeyValueNode(std::unique_ptr<Document> &D)
284 : Node(NK_KeyValue, D, StringRef(), StringRef()) {}
285
286 /// Parse and return the key.
287 ///
288 /// This may be called multiple times.
289 ///
290 /// \returns The key, or nullptr if failed() == true.
291 Node *getKey();
292
293 /// Parse and return the value.
294 ///
295 /// This may be called multiple times.
296 ///
297 /// \returns The value, or nullptr if failed() == true.
298 Node *getValue();
299
300 void skip() override {
301 if (Node *Key = getKey()) {
302 Key->skip();
303 if (Node *Val = getValue())
304 Val->skip();
305 }
306 }
307
308 static bool classof(const Node *N) {
309 return N->getType() == NK_KeyValue;
310 }
311
312private:
313 Node *Key = nullptr;
314 Node *Value = nullptr;
315};
316
317/// This is an iterator abstraction over YAML collections shared by both
318/// sequences and maps.
319///
320/// BaseT must have a ValueT* member named CurrentEntry and a member function
321/// increment() which must set CurrentEntry to 0 to create an end iterator.
322template <class BaseT, class ValueT>
323class basic_collection_iterator
324 : public std::iterator<std::input_iterator_tag, ValueT> {
325public:
326 basic_collection_iterator() = default;
327 basic_collection_iterator(BaseT *B) : Base(B) {}
328
329 ValueT *operator->() const {
330 assert(Base && Base->CurrentEntry && "Attempted to access end iterator!")((Base && Base->CurrentEntry && "Attempted to access end iterator!"
) ? static_cast<void> (0) : __assert_fail ("Base && Base->CurrentEntry && \"Attempted to access end iterator!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/Support/YAMLParser.h"
, 330, __PRETTY_FUNCTION__))
;
331 return Base->CurrentEntry;
332 }
333
334 ValueT &operator*() const {
335 assert(Base && Base->CurrentEntry &&((Base && Base->CurrentEntry && "Attempted to dereference end iterator!"
) ? static_cast<void> (0) : __assert_fail ("Base && Base->CurrentEntry && \"Attempted to dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/Support/YAMLParser.h"
, 336, __PRETTY_FUNCTION__))
336 "Attempted to dereference end iterator!")((Base && Base->CurrentEntry && "Attempted to dereference end iterator!"
) ? static_cast<void> (0) : __assert_fail ("Base && Base->CurrentEntry && \"Attempted to dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/Support/YAMLParser.h"
, 336, __PRETTY_FUNCTION__))
;
337 return *Base->CurrentEntry;
338 }
339
340 operator ValueT *() const {
341 assert(Base && Base->CurrentEntry && "Attempted to access end iterator!")((Base && Base->CurrentEntry && "Attempted to access end iterator!"
) ? static_cast<void> (0) : __assert_fail ("Base && Base->CurrentEntry && \"Attempted to access end iterator!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/Support/YAMLParser.h"
, 341, __PRETTY_FUNCTION__))
;
342 return Base->CurrentEntry;
343 }
344
345 /// Note on EqualityComparable:
346 ///
347 /// The iterator is not re-entrant,
348 /// it is meant to be used for parsing YAML on-demand
349 /// Once iteration started - it can point only to one entry at a time
350 /// hence Base.CurrentEntry and Other.Base.CurrentEntry are equal
351 /// iff Base and Other.Base are equal.
352 bool operator==(const basic_collection_iterator &Other) const {
353 if (Base && (Base == Other.Base)) {
354 assert((Base->CurrentEntry == Other.Base->CurrentEntry)(((Base->CurrentEntry == Other.Base->CurrentEntry) &&
"Equal Bases expected to point to equal Entries") ? static_cast
<void> (0) : __assert_fail ("(Base->CurrentEntry == Other.Base->CurrentEntry) && \"Equal Bases expected to point to equal Entries\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/Support/YAMLParser.h"
, 355, __PRETTY_FUNCTION__))
355 && "Equal Bases expected to point to equal Entries")(((Base->CurrentEntry == Other.Base->CurrentEntry) &&
"Equal Bases expected to point to equal Entries") ? static_cast
<void> (0) : __assert_fail ("(Base->CurrentEntry == Other.Base->CurrentEntry) && \"Equal Bases expected to point to equal Entries\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/Support/YAMLParser.h"
, 355, __PRETTY_FUNCTION__))
;
356 }
357
358 return Base == Other.Base;
359 }
360
361 bool operator!=(const basic_collection_iterator &Other) const {
362 return !(Base == Other.Base);
363 }
364
365 basic_collection_iterator &operator++() {
366 assert(Base && "Attempted to advance iterator past end!")((Base && "Attempted to advance iterator past end!") ?
static_cast<void> (0) : __assert_fail ("Base && \"Attempted to advance iterator past end!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/Support/YAMLParser.h"
, 366, __PRETTY_FUNCTION__))
;
367 Base->increment();
368 // Create an end iterator.
369 if (!Base->CurrentEntry)
370 Base = nullptr;
371 return *this;
372 }
373
374private:
375 BaseT *Base = nullptr;
376};
377
378// The following two templates are used for both MappingNode and Sequence Node.
379template <class CollectionType>
380typename CollectionType::iterator begin(CollectionType &C) {
381 assert(C.IsAtBeginning && "You may only iterate over a collection once!")((C.IsAtBeginning && "You may only iterate over a collection once!"
) ? static_cast<void> (0) : __assert_fail ("C.IsAtBeginning && \"You may only iterate over a collection once!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/Support/YAMLParser.h"
, 381, __PRETTY_FUNCTION__))
;
382 C.IsAtBeginning = false;
383 typename CollectionType::iterator ret(&C);
384 ++ret;
385 return ret;
386}
387
388template <class CollectionType> void skip(CollectionType &C) {
389 // TODO: support skipping from the middle of a parsed collection ;/
390 assert((C.IsAtBeginning || C.IsAtEnd) && "Cannot skip mid parse!")(((C.IsAtBeginning || C.IsAtEnd) && "Cannot skip mid parse!"
) ? static_cast<void> (0) : __assert_fail ("(C.IsAtBeginning || C.IsAtEnd) && \"Cannot skip mid parse!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/Support/YAMLParser.h"
, 390, __PRETTY_FUNCTION__))
;
391 if (C.IsAtBeginning)
392 for (typename CollectionType::iterator i = begin(C), e = C.end(); i != e;
393 ++i)
394 i->skip();
395}
396
397/// Represents a YAML map created from either a block map for a flow map.
398///
399/// This parses the YAML stream as increment() is called.
400///
401/// Example:
402/// Name: _main
403/// Scope: Global
404class MappingNode final : public Node {
405 void anchor() override;
406
407public:
408 enum MappingType {
409 MT_Block,
410 MT_Flow,
411 MT_Inline ///< An inline mapping node is used for "[key: value]".
412 };
413
414 MappingNode(std::unique_ptr<Document> &D, StringRef Anchor, StringRef Tag,
415 MappingType MT)
416 : Node(NK_Mapping, D, Anchor, Tag), Type(MT) {}
417
418 friend class basic_collection_iterator<MappingNode, KeyValueNode>;
419
420 using iterator = basic_collection_iterator<MappingNode, KeyValueNode>;
421
422 template <class T> friend typename T::iterator yaml::begin(T &);
423 template <class T> friend void yaml::skip(T &);
424
425 iterator begin() { return yaml::begin(*this); }
426
427 iterator end() { return iterator(); }
428
429 void skip() override { yaml::skip(*this); }
430
431 static bool classof(const Node *N) {
432 return N->getType() == NK_Mapping;
433 }
434
435private:
436 MappingType Type;
437 bool IsAtBeginning = true;
438 bool IsAtEnd = false;
439 KeyValueNode *CurrentEntry = nullptr;
440
441 void increment();
442};
443
444/// Represents a YAML sequence created from either a block sequence for a
445/// flow sequence.
446///
447/// This parses the YAML stream as increment() is called.
448///
449/// Example:
450/// - Hello
451/// - World
452class SequenceNode final : public Node {
453 void anchor() override;
454
455public:
456 enum SequenceType {
457 ST_Block,
458 ST_Flow,
459 // Use for:
460 //
461 // key:
462 // - val1
463 // - val2
464 //
465 // As a BlockMappingEntry and BlockEnd are not created in this case.
466 ST_Indentless
467 };
468
469 SequenceNode(std::unique_ptr<Document> &D, StringRef Anchor, StringRef Tag,
470 SequenceType ST)
471 : Node(NK_Sequence, D, Anchor, Tag), SeqType(ST) {}
472
473 friend class basic_collection_iterator<SequenceNode, Node>;
474
475 using iterator = basic_collection_iterator<SequenceNode, Node>;
476
477 template <class T> friend typename T::iterator yaml::begin(T &);
478 template <class T> friend void yaml::skip(T &);
479
480 void increment();
481
482 iterator begin() { return yaml::begin(*this); }
483
484 iterator end() { return iterator(); }
485
486 void skip() override { yaml::skip(*this); }
487
488 static bool classof(const Node *N) {
489 return N->getType() == NK_Sequence;
490 }
491
492private:
493 SequenceType SeqType;
494 bool IsAtBeginning = true;
495 bool IsAtEnd = false;
496 bool WasPreviousTokenFlowEntry = true; // Start with an imaginary ','.
497 Node *CurrentEntry = nullptr;
498};
499
500/// Represents an alias to a Node with an anchor.
501///
502/// Example:
503/// *AnchorName
504class AliasNode final : public Node {
505 void anchor() override;
506
507public:
508 AliasNode(std::unique_ptr<Document> &D, StringRef Val)
509 : Node(NK_Alias, D, StringRef(), StringRef()), Name(Val) {}
510
511 StringRef getName() const { return Name; }
512 Node *getTarget();
513
514 static bool classof(const Node *N) { return N->getType() == NK_Alias; }
515
516private:
517 StringRef Name;
518};
519
520/// A YAML Stream is a sequence of Documents. A document contains a root
521/// node.
522class Document {
523public:
524 Document(Stream &ParentStream);
525
526 /// Root for parsing a node. Returns a single node.
527 Node *parseBlockNode();
528
529 /// Finish parsing the current document and return true if there are
530 /// more. Return false otherwise.
531 bool skip();
532
533 /// Parse and return the root level node.
534 Node *getRoot() {
535 if (Root)
17
Assuming field 'Root' is non-null, which participates in a condition later
18
Taking true branch
536 return Root;
19
Returning pointer, which participates in a condition later
537 return Root = parseBlockNode();
538 }
539
540 const std::map<StringRef, StringRef> &getTagMap() const { return TagMap; }
541
542private:
543 friend class Node;
544 friend class document_iterator;
545
546 /// Stream to read tokens from.
547 Stream &stream;
548
549 /// Used to allocate nodes to. All are destroyed without calling their
550 /// destructor when the document is destroyed.
551 BumpPtrAllocator NodeAllocator;
552
553 /// The root node. Used to support skipping a partially parsed
554 /// document.
555 Node *Root;
556
557 /// Maps tag prefixes to their expansion.
558 std::map<StringRef, StringRef> TagMap;
559
560 Token &peekNext();
561 Token getNext();
562 void setError(const Twine &Message, Token &Location) const;
563 bool failed() const;
564
565 /// Parse %BLAH directives and return true if any were encountered.
566 bool parseDirectives();
567
568 /// Parse %YAML
569 void parseYAMLDirective();
570
571 /// Parse %TAG
572 void parseTAGDirective();
573
574 /// Consume the next token and error if it is not \a TK.
575 bool expectToken(int TK);
576};
577
578/// Iterator abstraction for Documents over a Stream.
579class document_iterator {
580public:
581 document_iterator() = default;
582 document_iterator(std::unique_ptr<Document> &D) : Doc(&D) {}
583
584 bool operator==(const document_iterator &Other) const {
585 if (isAtEnd() || Other.isAtEnd())
3
Calling 'document_iterator::isAtEnd'
6
Returning from 'document_iterator::isAtEnd'
7
Calling 'document_iterator::isAtEnd'
10
Returning from 'document_iterator::isAtEnd'
11
Taking false branch
586 return isAtEnd() && Other.isAtEnd();
587
588 return Doc == Other.Doc;
12
Assuming 'Doc' is not equal to 'Other.Doc'
13
Returning zero, which participates in a condition later
589 }
590 bool operator!=(const document_iterator &Other) const {
591 return !(*this == Other);
592 }
593
594 document_iterator operator++() {
595 assert(Doc && "incrementing iterator past the end.")((Doc && "incrementing iterator past the end.") ? static_cast
<void> (0) : __assert_fail ("Doc && \"incrementing iterator past the end.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/Support/YAMLParser.h"
, 595, __PRETTY_FUNCTION__))
;
596 if (!(*Doc)->skip()) {
597 Doc->reset(nullptr);
598 } else {
599 Stream &S = (*Doc)->stream;
600 Doc->reset(new Document(S));
601 }
602 return *this;
603 }
604
605 Document &operator*() { return *Doc->get(); }
606
607 std::unique_ptr<Document> &operator->() { return *Doc; }
608
609private:
610 bool isAtEnd() const { return !Doc || !*Doc; }
4
Assuming field 'Doc' is non-null
5
Returning zero, which participates in a condition later
8
Assuming field 'Doc' is non-null
9
Returning zero, which participates in a condition later
611
612 std::unique_ptr<Document> *Doc = nullptr;
613};
614
615} // end namespace yaml
616
617} // end namespace llvm
618
619#endif // LLVM_SUPPORT_YAMLPARSER_H