Bug Summary

File:tools/clang/lib/AST/CommentLexer.cpp
Warning:line 862, column 14
Dereference of null pointer (loaded from variable 'Invalid')

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name CommentLexer.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -mrelocation-model pic -pic-level 2 -mthread-model posix -relaxed-aliasing -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -momit-leaf-frame-pointer -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-8/lib/clang/8.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/tools/clang/lib/AST -I /build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST -I /build/llvm-toolchain-snapshot-8~svn345461/tools/clang/include -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/tools/clang/include -I /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/include -I /build/llvm-toolchain-snapshot-8~svn345461/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/include/clang/8.0.0/include/ -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-8/lib/clang/8.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++11 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-8~svn345461/build-llvm/tools/clang/lib/AST -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -fobjc-runtime=gcc -fno-common -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -o /tmp/scan-build-2018-10-27-211344-32123-1 -x c++ /build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp -faddrsig
1//===--- CommentLexer.cpp -------------------------------------------------===//
2//
3// The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9
10#include "clang/AST/CommentLexer.h"
11#include "clang/AST/CommentCommandTraits.h"
12#include "clang/AST/CommentDiagnostic.h"
13#include "clang/Basic/CharInfo.h"
14#include "llvm/ADT/StringExtras.h"
15#include "llvm/ADT/StringSwitch.h"
16#include "llvm/Support/ConvertUTF.h"
17#include "llvm/Support/ErrorHandling.h"
18
19namespace clang {
20namespace comments {
21
22void Token::dump(const Lexer &L, const SourceManager &SM) const {
23 llvm::errs() << "comments::Token Kind=" << Kind << " ";
24 Loc.print(llvm::errs(), SM);
25 llvm::errs() << " " << Length << " \"" << L.getSpelling(*this, SM) << "\"\n";
1
Passing null pointer value via 3rd parameter 'Invalid'
2
Calling 'Lexer::getSpelling'
26}
27
28static inline bool isHTMLNamedCharacterReferenceCharacter(char C) {
29 return isLetter(C);
30}
31
32static inline bool isHTMLDecimalCharacterReferenceCharacter(char C) {
33 return isDigit(C);
34}
35
36static inline bool isHTMLHexCharacterReferenceCharacter(char C) {
37 return isHexDigit(C);
38}
39
40static inline StringRef convertCodePointToUTF8(
41 llvm::BumpPtrAllocator &Allocator,
42 unsigned CodePoint) {
43 char *Resolved = Allocator.Allocate<char>(UNI_MAX_UTF8_BYTES_PER_CODE_POINT4);
44 char *ResolvedPtr = Resolved;
45 if (llvm::ConvertCodePointToUTF8(CodePoint, ResolvedPtr))
46 return StringRef(Resolved, ResolvedPtr - Resolved);
47 else
48 return StringRef();
49}
50
51namespace {
52
53#include "clang/AST/CommentHTMLTags.inc"
54#include "clang/AST/CommentHTMLNamedCharacterReferences.inc"
55
56} // end anonymous namespace
57
58StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
59 // Fast path, first check a few most widely used named character references.
60 return llvm::StringSwitch<StringRef>(Name)
61 .Case("amp", "&")
62 .Case("lt", "<")
63 .Case("gt", ">")
64 .Case("quot", "\"")
65 .Case("apos", "\'")
66 // Slow path.
67 .Default(translateHTMLNamedCharacterReferenceToUTF8(Name));
68}
69
70StringRef Lexer::resolveHTMLDecimalCharacterReference(StringRef Name) const {
71 unsigned CodePoint = 0;
72 for (unsigned i = 0, e = Name.size(); i != e; ++i) {
73 assert(isHTMLDecimalCharacterReferenceCharacter(Name[i]))((isHTMLDecimalCharacterReferenceCharacter(Name[i])) ? static_cast
<void> (0) : __assert_fail ("isHTMLDecimalCharacterReferenceCharacter(Name[i])"
, "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 73, __PRETTY_FUNCTION__))
;
74 CodePoint *= 10;
75 CodePoint += Name[i] - '0';
76 }
77 return convertCodePointToUTF8(Allocator, CodePoint);
78}
79
80StringRef Lexer::resolveHTMLHexCharacterReference(StringRef Name) const {
81 unsigned CodePoint = 0;
82 for (unsigned i = 0, e = Name.size(); i != e; ++i) {
83 CodePoint *= 16;
84 const char C = Name[i];
85 assert(isHTMLHexCharacterReferenceCharacter(C))((isHTMLHexCharacterReferenceCharacter(C)) ? static_cast<void
> (0) : __assert_fail ("isHTMLHexCharacterReferenceCharacter(C)"
, "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 85, __PRETTY_FUNCTION__))
;
86 CodePoint += llvm::hexDigitValue(C);
87 }
88 return convertCodePointToUTF8(Allocator, CodePoint);
89}
90
91void Lexer::skipLineStartingDecorations() {
92 // This function should be called only for C comments
93 assert(CommentState == LCS_InsideCComment)((CommentState == LCS_InsideCComment) ? static_cast<void>
(0) : __assert_fail ("CommentState == LCS_InsideCComment", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 93, __PRETTY_FUNCTION__))
;
94
95 if (BufferPtr == CommentEnd)
96 return;
97
98 switch (*BufferPtr) {
99 case ' ':
100 case '\t':
101 case '\f':
102 case '\v': {
103 const char *NewBufferPtr = BufferPtr;
104 NewBufferPtr++;
105 if (NewBufferPtr == CommentEnd)
106 return;
107
108 char C = *NewBufferPtr;
109 while (isHorizontalWhitespace(C)) {
110 NewBufferPtr++;
111 if (NewBufferPtr == CommentEnd)
112 return;
113 C = *NewBufferPtr;
114 }
115 if (C == '*')
116 BufferPtr = NewBufferPtr + 1;
117 break;
118 }
119 case '*':
120 BufferPtr++;
121 break;
122 }
123}
124
125namespace {
126/// Returns pointer to the first newline character in the string.
127const char *findNewline(const char *BufferPtr, const char *BufferEnd) {
128 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
129 if (isVerticalWhitespace(*BufferPtr))
130 return BufferPtr;
131 }
132 return BufferEnd;
133}
134
135const char *skipNewline(const char *BufferPtr, const char *BufferEnd) {
136 if (BufferPtr == BufferEnd)
137 return BufferPtr;
138
139 if (*BufferPtr == '\n')
140 BufferPtr++;
141 else {
142 assert(*BufferPtr == '\r')((*BufferPtr == '\r') ? static_cast<void> (0) : __assert_fail
("*BufferPtr == '\\r'", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 142, __PRETTY_FUNCTION__))
;
143 BufferPtr++;
144 if (BufferPtr != BufferEnd && *BufferPtr == '\n')
145 BufferPtr++;
146 }
147 return BufferPtr;
148}
149
150const char *skipNamedCharacterReference(const char *BufferPtr,
151 const char *BufferEnd) {
152 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
153 if (!isHTMLNamedCharacterReferenceCharacter(*BufferPtr))
154 return BufferPtr;
155 }
156 return BufferEnd;
157}
158
159const char *skipDecimalCharacterReference(const char *BufferPtr,
160 const char *BufferEnd) {
161 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
162 if (!isHTMLDecimalCharacterReferenceCharacter(*BufferPtr))
163 return BufferPtr;
164 }
165 return BufferEnd;
166}
167
168const char *skipHexCharacterReference(const char *BufferPtr,
169 const char *BufferEnd) {
170 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
171 if (!isHTMLHexCharacterReferenceCharacter(*BufferPtr))
172 return BufferPtr;
173 }
174 return BufferEnd;
175}
176
177bool isHTMLIdentifierStartingCharacter(char C) {
178 return isLetter(C);
179}
180
181bool isHTMLIdentifierCharacter(char C) {
182 return isAlphanumeric(C);
183}
184
185const char *skipHTMLIdentifier(const char *BufferPtr, const char *BufferEnd) {
186 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
187 if (!isHTMLIdentifierCharacter(*BufferPtr))
188 return BufferPtr;
189 }
190 return BufferEnd;
191}
192
193/// Skip HTML string quoted in single or double quotes. Escaping quotes inside
194/// string allowed.
195///
196/// Returns pointer to closing quote.
197const char *skipHTMLQuotedString(const char *BufferPtr, const char *BufferEnd)
198{
199 const char Quote = *BufferPtr;
200 assert(Quote == '\"' || Quote == '\'')((Quote == '\"' || Quote == '\'') ? static_cast<void> (
0) : __assert_fail ("Quote == '\\\"' || Quote == '\\''", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 200, __PRETTY_FUNCTION__))
;
201
202 BufferPtr++;
203 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
204 const char C = *BufferPtr;
205 if (C == Quote && BufferPtr[-1] != '\\')
206 return BufferPtr;
207 }
208 return BufferEnd;
209}
210
211const char *skipWhitespace(const char *BufferPtr, const char *BufferEnd) {
212 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
213 if (!isWhitespace(*BufferPtr))
214 return BufferPtr;
215 }
216 return BufferEnd;
217}
218
219bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
220 return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
221}
222
223bool isCommandNameStartCharacter(char C) {
224 return isLetter(C);
225}
226
227bool isCommandNameCharacter(char C) {
228 return isAlphanumeric(C);
229}
230
231const char *skipCommandName(const char *BufferPtr, const char *BufferEnd) {
232 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
233 if (!isCommandNameCharacter(*BufferPtr))
234 return BufferPtr;
235 }
236 return BufferEnd;
237}
238
239/// Return the one past end pointer for BCPL comments.
240/// Handles newlines escaped with backslash or trigraph for backslahs.
241const char *findBCPLCommentEnd(const char *BufferPtr, const char *BufferEnd) {
242 const char *CurPtr = BufferPtr;
243 while (CurPtr != BufferEnd) {
244 while (!isVerticalWhitespace(*CurPtr)) {
245 CurPtr++;
246 if (CurPtr == BufferEnd)
247 return BufferEnd;
248 }
249 // We found a newline, check if it is escaped.
250 const char *EscapePtr = CurPtr - 1;
251 while(isHorizontalWhitespace(*EscapePtr))
252 EscapePtr--;
253
254 if (*EscapePtr == '\\' ||
255 (EscapePtr - 2 >= BufferPtr && EscapePtr[0] == '/' &&
256 EscapePtr[-1] == '?' && EscapePtr[-2] == '?')) {
257 // We found an escaped newline.
258 CurPtr = skipNewline(CurPtr, BufferEnd);
259 } else
260 return CurPtr; // Not an escaped newline.
261 }
262 return BufferEnd;
263}
264
265/// Return the one past end pointer for C comments.
266/// Very dumb, does not handle escaped newlines or trigraphs.
267const char *findCCommentEnd(const char *BufferPtr, const char *BufferEnd) {
268 for ( ; BufferPtr != BufferEnd; ++BufferPtr) {
269 if (*BufferPtr == '*') {
270 assert(BufferPtr + 1 != BufferEnd)((BufferPtr + 1 != BufferEnd) ? static_cast<void> (0) :
__assert_fail ("BufferPtr + 1 != BufferEnd", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 270, __PRETTY_FUNCTION__))
;
271 if (*(BufferPtr + 1) == '/')
272 return BufferPtr;
273 }
274 }
275 llvm_unreachable("buffer end hit before '*/' was seen")::llvm::llvm_unreachable_internal("buffer end hit before '*/' was seen"
, "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 275)
;
276}
277
278} // end anonymous namespace
279
280void Lexer::formTokenWithChars(Token &Result, const char *TokEnd,
281 tok::TokenKind Kind) {
282 const unsigned TokLen = TokEnd - BufferPtr;
283 Result.setLocation(getSourceLocation(BufferPtr));
284 Result.setKind(Kind);
285 Result.setLength(TokLen);
286#ifndef NDEBUG
287 Result.TextPtr = "<UNSET>";
288 Result.IntVal = 7;
289#endif
290 BufferPtr = TokEnd;
291}
292
293void Lexer::lexCommentText(Token &T) {
294 assert(CommentState == LCS_InsideBCPLComment ||((CommentState == LCS_InsideBCPLComment || CommentState == LCS_InsideCComment
) ? static_cast<void> (0) : __assert_fail ("CommentState == LCS_InsideBCPLComment || CommentState == LCS_InsideCComment"
, "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 295, __PRETTY_FUNCTION__))
295 CommentState == LCS_InsideCComment)((CommentState == LCS_InsideBCPLComment || CommentState == LCS_InsideCComment
) ? static_cast<void> (0) : __assert_fail ("CommentState == LCS_InsideBCPLComment || CommentState == LCS_InsideCComment"
, "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 295, __PRETTY_FUNCTION__))
;
296
297 // Handles lexing non-command text, i.e. text and newline.
298 auto HandleNonCommandToken = [&]() -> void {
299 assert(State == LS_Normal)((State == LS_Normal) ? static_cast<void> (0) : __assert_fail
("State == LS_Normal", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 299, __PRETTY_FUNCTION__))
;
300
301 const char *TokenPtr = BufferPtr;
302 assert(TokenPtr < CommentEnd)((TokenPtr < CommentEnd) ? static_cast<void> (0) : __assert_fail
("TokenPtr < CommentEnd", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 302, __PRETTY_FUNCTION__))
;
303 switch (*TokenPtr) {
304 case '\n':
305 case '\r':
306 TokenPtr = skipNewline(TokenPtr, CommentEnd);
307 formTokenWithChars(T, TokenPtr, tok::newline);
308
309 if (CommentState == LCS_InsideCComment)
310 skipLineStartingDecorations();
311 return;
312
313 default: {
314 StringRef TokStartSymbols = ParseCommands ? "\n\r\\@&<" : "\n\r";
315 size_t End = StringRef(TokenPtr, CommentEnd - TokenPtr)
316 .find_first_of(TokStartSymbols);
317 if (End != StringRef::npos)
318 TokenPtr += End;
319 else
320 TokenPtr = CommentEnd;
321 formTextToken(T, TokenPtr);
322 return;
323 }
324 }
325 };
326
327 if (!ParseCommands)
328 return HandleNonCommandToken();
329
330 switch (State) {
331 case LS_Normal:
332 break;
333 case LS_VerbatimBlockFirstLine:
334 lexVerbatimBlockFirstLine(T);
335 return;
336 case LS_VerbatimBlockBody:
337 lexVerbatimBlockBody(T);
338 return;
339 case LS_VerbatimLineText:
340 lexVerbatimLineText(T);
341 return;
342 case LS_HTMLStartTag:
343 lexHTMLStartTag(T);
344 return;
345 case LS_HTMLEndTag:
346 lexHTMLEndTag(T);
347 return;
348 }
349
350 assert(State == LS_Normal)((State == LS_Normal) ? static_cast<void> (0) : __assert_fail
("State == LS_Normal", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 350, __PRETTY_FUNCTION__))
;
351 const char *TokenPtr = BufferPtr;
352 assert(TokenPtr < CommentEnd)((TokenPtr < CommentEnd) ? static_cast<void> (0) : __assert_fail
("TokenPtr < CommentEnd", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 352, __PRETTY_FUNCTION__))
;
353 switch(*TokenPtr) {
354 case '\\':
355 case '@': {
356 // Commands that start with a backslash and commands that start with
357 // 'at' have equivalent semantics. But we keep information about the
358 // exact syntax in AST for comments.
359 tok::TokenKind CommandKind =
360 (*TokenPtr == '@') ? tok::at_command : tok::backslash_command;
361 TokenPtr++;
362 if (TokenPtr == CommentEnd) {
363 formTextToken(T, TokenPtr);
364 return;
365 }
366 char C = *TokenPtr;
367 switch (C) {
368 default:
369 break;
370
371 case '\\': case '@': case '&': case '$':
372 case '#': case '<': case '>': case '%':
373 case '\"': case '.': case ':':
374 // This is one of \\ \@ \& \$ etc escape sequences.
375 TokenPtr++;
376 if (C == ':' && TokenPtr != CommentEnd && *TokenPtr == ':') {
377 // This is the \:: escape sequence.
378 TokenPtr++;
379 }
380 StringRef UnescapedText(BufferPtr + 1, TokenPtr - (BufferPtr + 1));
381 formTokenWithChars(T, TokenPtr, tok::text);
382 T.setText(UnescapedText);
383 return;
384 }
385
386 // Don't make zero-length commands.
387 if (!isCommandNameStartCharacter(*TokenPtr)) {
388 formTextToken(T, TokenPtr);
389 return;
390 }
391
392 TokenPtr = skipCommandName(TokenPtr, CommentEnd);
393 unsigned Length = TokenPtr - (BufferPtr + 1);
394
395 // Hardcoded support for lexing LaTeX formula commands
396 // \f$ \f[ \f] \f{ \f} as a single command.
397 if (Length == 1 && TokenPtr[-1] == 'f' && TokenPtr != CommentEnd) {
398 C = *TokenPtr;
399 if (C == '$' || C == '[' || C == ']' || C == '{' || C == '}') {
400 TokenPtr++;
401 Length++;
402 }
403 }
404
405 StringRef CommandName(BufferPtr + 1, Length);
406
407 const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
408 if (!Info) {
409 if ((Info = Traits.getTypoCorrectCommandInfo(CommandName))) {
410 StringRef CorrectedName = Info->Name;
411 SourceLocation Loc = getSourceLocation(BufferPtr);
412 SourceLocation EndLoc = getSourceLocation(TokenPtr);
413 SourceRange FullRange = SourceRange(Loc, EndLoc);
414 SourceRange CommandRange(Loc.getLocWithOffset(1), EndLoc);
415 Diag(Loc, diag::warn_correct_comment_command_name)
416 << FullRange << CommandName << CorrectedName
417 << FixItHint::CreateReplacement(CommandRange, CorrectedName);
418 } else {
419 formTokenWithChars(T, TokenPtr, tok::unknown_command);
420 T.setUnknownCommandName(CommandName);
421 Diag(T.getLocation(), diag::warn_unknown_comment_command_name)
422 << SourceRange(T.getLocation(), T.getEndLocation());
423 return;
424 }
425 }
426 if (Info->IsVerbatimBlockCommand) {
427 setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
428 return;
429 }
430 if (Info->IsVerbatimLineCommand) {
431 setupAndLexVerbatimLine(T, TokenPtr, Info);
432 return;
433 }
434 formTokenWithChars(T, TokenPtr, CommandKind);
435 T.setCommandID(Info->getID());
436 return;
437 }
438
439 case '&':
440 lexHTMLCharacterReference(T);
441 return;
442
443 case '<': {
444 TokenPtr++;
445 if (TokenPtr == CommentEnd) {
446 formTextToken(T, TokenPtr);
447 return;
448 }
449 const char C = *TokenPtr;
450 if (isHTMLIdentifierStartingCharacter(C))
451 setupAndLexHTMLStartTag(T);
452 else if (C == '/')
453 setupAndLexHTMLEndTag(T);
454 else
455 formTextToken(T, TokenPtr);
456 return;
457 }
458
459 default:
460 return HandleNonCommandToken();
461 }
462}
463
464void Lexer::setupAndLexVerbatimBlock(Token &T,
465 const char *TextBegin,
466 char Marker, const CommandInfo *Info) {
467 assert(Info->IsVerbatimBlockCommand)((Info->IsVerbatimBlockCommand) ? static_cast<void> (
0) : __assert_fail ("Info->IsVerbatimBlockCommand", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 467, __PRETTY_FUNCTION__))
;
468
469 VerbatimBlockEndCommandName.clear();
470 VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
471 VerbatimBlockEndCommandName.append(Info->EndCommandName);
472
473 formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
474 T.setVerbatimBlockID(Info->getID());
475
476 // If there is a newline following the verbatim opening command, skip the
477 // newline so that we don't create an tok::verbatim_block_line with empty
478 // text content.
479 if (BufferPtr != CommentEnd &&
480 isVerticalWhitespace(*BufferPtr)) {
481 BufferPtr = skipNewline(BufferPtr, CommentEnd);
482 State = LS_VerbatimBlockBody;
483 return;
484 }
485
486 State = LS_VerbatimBlockFirstLine;
487}
488
489void Lexer::lexVerbatimBlockFirstLine(Token &T) {
490again:
491 assert(BufferPtr < CommentEnd)((BufferPtr < CommentEnd) ? static_cast<void> (0) : __assert_fail
("BufferPtr < CommentEnd", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 491, __PRETTY_FUNCTION__))
;
492
493 // FIXME: It would be better to scan the text once, finding either the block
494 // end command or newline.
495 //
496 // Extract current line.
497 const char *Newline = findNewline(BufferPtr, CommentEnd);
498 StringRef Line(BufferPtr, Newline - BufferPtr);
499
500 // Look for end command in current line.
501 size_t Pos = Line.find(VerbatimBlockEndCommandName);
502 const char *TextEnd;
503 const char *NextLine;
504 if (Pos == StringRef::npos) {
505 // Current line is completely verbatim.
506 TextEnd = Newline;
507 NextLine = skipNewline(Newline, CommentEnd);
508 } else if (Pos == 0) {
509 // Current line contains just an end command.
510 const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
511 StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
512 formTokenWithChars(T, End, tok::verbatim_block_end);
513 T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
514 State = LS_Normal;
515 return;
516 } else {
517 // There is some text, followed by end command. Extract text first.
518 TextEnd = BufferPtr + Pos;
519 NextLine = TextEnd;
520 // If there is only whitespace before end command, skip whitespace.
521 if (isWhitespace(BufferPtr, TextEnd)) {
522 BufferPtr = TextEnd;
523 goto again;
524 }
525 }
526
527 StringRef Text(BufferPtr, TextEnd - BufferPtr);
528 formTokenWithChars(T, NextLine, tok::verbatim_block_line);
529 T.setVerbatimBlockText(Text);
530
531 State = LS_VerbatimBlockBody;
532}
533
534void Lexer::lexVerbatimBlockBody(Token &T) {
535 assert(State == LS_VerbatimBlockBody)((State == LS_VerbatimBlockBody) ? static_cast<void> (0
) : __assert_fail ("State == LS_VerbatimBlockBody", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 535, __PRETTY_FUNCTION__))
;
536
537 if (CommentState == LCS_InsideCComment)
538 skipLineStartingDecorations();
539
540 if (BufferPtr == CommentEnd) {
541 formTokenWithChars(T, BufferPtr, tok::verbatim_block_line);
542 T.setVerbatimBlockText("");
543 return;
544 }
545
546 lexVerbatimBlockFirstLine(T);
547}
548
549void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
550 const CommandInfo *Info) {
551 assert(Info->IsVerbatimLineCommand)((Info->IsVerbatimLineCommand) ? static_cast<void> (
0) : __assert_fail ("Info->IsVerbatimLineCommand", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 551, __PRETTY_FUNCTION__))
;
552 formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
553 T.setVerbatimLineID(Info->getID());
554
555 State = LS_VerbatimLineText;
556}
557
558void Lexer::lexVerbatimLineText(Token &T) {
559 assert(State == LS_VerbatimLineText)((State == LS_VerbatimLineText) ? static_cast<void> (0)
: __assert_fail ("State == LS_VerbatimLineText", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 559, __PRETTY_FUNCTION__))
;
560
561 // Extract current line.
562 const char *Newline = findNewline(BufferPtr, CommentEnd);
563 StringRef Text(BufferPtr, Newline - BufferPtr);
564 formTokenWithChars(T, Newline, tok::verbatim_line_text);
565 T.setVerbatimLineText(Text);
566
567 State = LS_Normal;
568}
569
570void Lexer::lexHTMLCharacterReference(Token &T) {
571 const char *TokenPtr = BufferPtr;
572 assert(*TokenPtr == '&')((*TokenPtr == '&') ? static_cast<void> (0) : __assert_fail
("*TokenPtr == '&'", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 572, __PRETTY_FUNCTION__))
;
573 TokenPtr++;
574 if (TokenPtr == CommentEnd) {
575 formTextToken(T, TokenPtr);
576 return;
577 }
578 const char *NamePtr;
579 bool isNamed = false;
580 bool isDecimal = false;
581 char C = *TokenPtr;
582 if (isHTMLNamedCharacterReferenceCharacter(C)) {
583 NamePtr = TokenPtr;
584 TokenPtr = skipNamedCharacterReference(TokenPtr, CommentEnd);
585 isNamed = true;
586 } else if (C == '#') {
587 TokenPtr++;
588 if (TokenPtr == CommentEnd) {
589 formTextToken(T, TokenPtr);
590 return;
591 }
592 C = *TokenPtr;
593 if (isHTMLDecimalCharacterReferenceCharacter(C)) {
594 NamePtr = TokenPtr;
595 TokenPtr = skipDecimalCharacterReference(TokenPtr, CommentEnd);
596 isDecimal = true;
597 } else if (C == 'x' || C == 'X') {
598 TokenPtr++;
599 NamePtr = TokenPtr;
600 TokenPtr = skipHexCharacterReference(TokenPtr, CommentEnd);
601 } else {
602 formTextToken(T, TokenPtr);
603 return;
604 }
605 } else {
606 formTextToken(T, TokenPtr);
607 return;
608 }
609 if (NamePtr == TokenPtr || TokenPtr == CommentEnd ||
610 *TokenPtr != ';') {
611 formTextToken(T, TokenPtr);
612 return;
613 }
614 StringRef Name(NamePtr, TokenPtr - NamePtr);
615 TokenPtr++; // Skip semicolon.
616 StringRef Resolved;
617 if (isNamed)
618 Resolved = resolveHTMLNamedCharacterReference(Name);
619 else if (isDecimal)
620 Resolved = resolveHTMLDecimalCharacterReference(Name);
621 else
622 Resolved = resolveHTMLHexCharacterReference(Name);
623
624 if (Resolved.empty()) {
625 formTextToken(T, TokenPtr);
626 return;
627 }
628 formTokenWithChars(T, TokenPtr, tok::text);
629 T.setText(Resolved);
630}
631
632void Lexer::setupAndLexHTMLStartTag(Token &T) {
633 assert(BufferPtr[0] == '<' &&((BufferPtr[0] == '<' && isHTMLIdentifierStartingCharacter
(BufferPtr[1])) ? static_cast<void> (0) : __assert_fail
("BufferPtr[0] == '<' && isHTMLIdentifierStartingCharacter(BufferPtr[1])"
, "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 634, __PRETTY_FUNCTION__))
634 isHTMLIdentifierStartingCharacter(BufferPtr[1]))((BufferPtr[0] == '<' && isHTMLIdentifierStartingCharacter
(BufferPtr[1])) ? static_cast<void> (0) : __assert_fail
("BufferPtr[0] == '<' && isHTMLIdentifierStartingCharacter(BufferPtr[1])"
, "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 634, __PRETTY_FUNCTION__))
;
635 const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
636 StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
637 if (!isHTMLTagName(Name)) {
638 formTextToken(T, TagNameEnd);
639 return;
640 }
641
642 formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
643 T.setHTMLTagStartName(Name);
644
645 BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
646
647 const char C = *BufferPtr;
648 if (BufferPtr != CommentEnd &&
649 (C == '>' || C == '/' || isHTMLIdentifierStartingCharacter(C)))
650 State = LS_HTMLStartTag;
651}
652
653void Lexer::lexHTMLStartTag(Token &T) {
654 assert(State == LS_HTMLStartTag)((State == LS_HTMLStartTag) ? static_cast<void> (0) : __assert_fail
("State == LS_HTMLStartTag", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 654, __PRETTY_FUNCTION__))
;
655
656 const char *TokenPtr = BufferPtr;
657 char C = *TokenPtr;
658 if (isHTMLIdentifierCharacter(C)) {
659 TokenPtr = skipHTMLIdentifier(TokenPtr, CommentEnd);
660 StringRef Ident(BufferPtr, TokenPtr - BufferPtr);
661 formTokenWithChars(T, TokenPtr, tok::html_ident);
662 T.setHTMLIdent(Ident);
663 } else {
664 switch (C) {
665 case '=':
666 TokenPtr++;
667 formTokenWithChars(T, TokenPtr, tok::html_equals);
668 break;
669 case '\"':
670 case '\'': {
671 const char *OpenQuote = TokenPtr;
672 TokenPtr = skipHTMLQuotedString(TokenPtr, CommentEnd);
673 const char *ClosingQuote = TokenPtr;
674 if (TokenPtr != CommentEnd) // Skip closing quote.
675 TokenPtr++;
676 formTokenWithChars(T, TokenPtr, tok::html_quoted_string);
677 T.setHTMLQuotedString(StringRef(OpenQuote + 1,
678 ClosingQuote - (OpenQuote + 1)));
679 break;
680 }
681 case '>':
682 TokenPtr++;
683 formTokenWithChars(T, TokenPtr, tok::html_greater);
684 State = LS_Normal;
685 return;
686 case '/':
687 TokenPtr++;
688 if (TokenPtr != CommentEnd && *TokenPtr == '>') {
689 TokenPtr++;
690 formTokenWithChars(T, TokenPtr, tok::html_slash_greater);
691 } else
692 formTextToken(T, TokenPtr);
693
694 State = LS_Normal;
695 return;
696 }
697 }
698
699 // Now look ahead and return to normal state if we don't see any HTML tokens
700 // ahead.
701 BufferPtr = skipWhitespace(BufferPtr, CommentEnd);
702 if (BufferPtr == CommentEnd) {
703 State = LS_Normal;
704 return;
705 }
706
707 C = *BufferPtr;
708 if (!isHTMLIdentifierStartingCharacter(C) &&
709 C != '=' && C != '\"' && C != '\'' && C != '>') {
710 State = LS_Normal;
711 return;
712 }
713}
714
715void Lexer::setupAndLexHTMLEndTag(Token &T) {
716 assert(BufferPtr[0] == '<' && BufferPtr[1] == '/')((BufferPtr[0] == '<' && BufferPtr[1] == '/') ? static_cast
<void> (0) : __assert_fail ("BufferPtr[0] == '<' && BufferPtr[1] == '/'"
, "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 716, __PRETTY_FUNCTION__))
;
717
718 const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
719 const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
720 StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
721 if (!isHTMLTagName(Name)) {
722 formTextToken(T, TagNameEnd);
723 return;
724 }
725
726 const char *End = skipWhitespace(TagNameEnd, CommentEnd);
727
728 formTokenWithChars(T, End, tok::html_end_tag);
729 T.setHTMLTagEndName(Name);
730
731 if (BufferPtr != CommentEnd && *BufferPtr == '>')
732 State = LS_HTMLEndTag;
733}
734
735void Lexer::lexHTMLEndTag(Token &T) {
736 assert(BufferPtr != CommentEnd && *BufferPtr == '>')((BufferPtr != CommentEnd && *BufferPtr == '>') ? static_cast
<void> (0) : __assert_fail ("BufferPtr != CommentEnd && *BufferPtr == '>'"
, "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 736, __PRETTY_FUNCTION__))
;
737
738 formTokenWithChars(T, BufferPtr + 1, tok::html_greater);
739 State = LS_Normal;
740}
741
742Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, DiagnosticsEngine &Diags,
743 const CommandTraits &Traits, SourceLocation FileLoc,
744 const char *BufferStart, const char *BufferEnd,
745 bool ParseCommands)
746 : Allocator(Allocator), Diags(Diags), Traits(Traits),
747 BufferStart(BufferStart), BufferEnd(BufferEnd), FileLoc(FileLoc),
748 BufferPtr(BufferStart), CommentState(LCS_BeforeComment), State(LS_Normal),
749 ParseCommands(ParseCommands) {}
750
751void Lexer::lex(Token &T) {
752again:
753 switch (CommentState) {
754 case LCS_BeforeComment:
755 if (BufferPtr == BufferEnd) {
756 formTokenWithChars(T, BufferPtr, tok::eof);
757 return;
758 }
759
760 assert(*BufferPtr == '/')((*BufferPtr == '/') ? static_cast<void> (0) : __assert_fail
("*BufferPtr == '/'", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 760, __PRETTY_FUNCTION__))
;
761 BufferPtr++; // Skip first slash.
762 switch(*BufferPtr) {
763 case '/': { // BCPL comment.
764 BufferPtr++; // Skip second slash.
765
766 if (BufferPtr != BufferEnd) {
767 // Skip Doxygen magic marker, if it is present.
768 // It might be missing because of a typo //< or /*<, or because we
769 // merged this non-Doxygen comment into a bunch of Doxygen comments
770 // around it: /** ... */ /* ... */ /** ... */
771 const char C = *BufferPtr;
772 if (C == '/' || C == '!')
773 BufferPtr++;
774 }
775
776 // Skip less-than symbol that marks trailing comments.
777 // Skip it even if the comment is not a Doxygen one, because //< and /*<
778 // are frequent typos.
779 if (BufferPtr != BufferEnd && *BufferPtr == '<')
780 BufferPtr++;
781
782 CommentState = LCS_InsideBCPLComment;
783 if (State != LS_VerbatimBlockBody && State != LS_VerbatimBlockFirstLine)
784 State = LS_Normal;
785 CommentEnd = findBCPLCommentEnd(BufferPtr, BufferEnd);
786 goto again;
787 }
788 case '*': { // C comment.
789 BufferPtr++; // Skip star.
790
791 // Skip Doxygen magic marker.
792 const char C = *BufferPtr;
793 if ((C == '*' && *(BufferPtr + 1) != '/') || C == '!')
794 BufferPtr++;
795
796 // Skip less-than symbol that marks trailing comments.
797 if (BufferPtr != BufferEnd && *BufferPtr == '<')
798 BufferPtr++;
799
800 CommentState = LCS_InsideCComment;
801 State = LS_Normal;
802 CommentEnd = findCCommentEnd(BufferPtr, BufferEnd);
803 goto again;
804 }
805 default:
806 llvm_unreachable("second character of comment should be '/' or '*'")::llvm::llvm_unreachable_internal("second character of comment should be '/' or '*'"
, "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 806)
;
807 }
808
809 case LCS_BetweenComments: {
810 // Consecutive comments are extracted only if there is only whitespace
811 // between them. So we can search for the start of the next comment.
812 const char *EndWhitespace = BufferPtr;
813 while(EndWhitespace != BufferEnd && *EndWhitespace != '/')
814 EndWhitespace++;
815
816 // Turn any whitespace between comments (and there is only whitespace
817 // between them -- guaranteed by comment extraction) into a newline. We
818 // have two newlines between C comments in total (first one was synthesized
819 // after a comment).
820 formTokenWithChars(T, EndWhitespace, tok::newline);
821
822 CommentState = LCS_BeforeComment;
823 break;
824 }
825
826 case LCS_InsideBCPLComment:
827 case LCS_InsideCComment:
828 if (BufferPtr != CommentEnd) {
829 lexCommentText(T);
830 break;
831 } else {
832 // Skip C comment closing sequence.
833 if (CommentState == LCS_InsideCComment) {
834 assert(BufferPtr[0] == '*' && BufferPtr[1] == '/')((BufferPtr[0] == '*' && BufferPtr[1] == '/') ? static_cast
<void> (0) : __assert_fail ("BufferPtr[0] == '*' && BufferPtr[1] == '/'"
, "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 834, __PRETTY_FUNCTION__))
;
835 BufferPtr += 2;
836 assert(BufferPtr <= BufferEnd)((BufferPtr <= BufferEnd) ? static_cast<void> (0) : __assert_fail
("BufferPtr <= BufferEnd", "/build/llvm-toolchain-snapshot-8~svn345461/tools/clang/lib/AST/CommentLexer.cpp"
, 836, __PRETTY_FUNCTION__))
;
837
838 // Synthenize newline just after the C comment, regardless if there is
839 // actually a newline.
840 formTokenWithChars(T, BufferPtr, tok::newline);
841
842 CommentState = LCS_BetweenComments;
843 break;
844 } else {
845 // Don't synthesized a newline after BCPL comment.
846 CommentState = LCS_BetweenComments;
847 goto again;
848 }
849 }
850 }
851}
852
853StringRef Lexer::getSpelling(const Token &Tok,
854 const SourceManager &SourceMgr,
855 bool *Invalid) const {
856 SourceLocation Loc = Tok.getLocation();
857 std::pair<FileID, unsigned> LocInfo = SourceMgr.getDecomposedLoc(Loc);
858
859 bool InvalidTemp = false;
860 StringRef File = SourceMgr.getBufferData(LocInfo.first, &InvalidTemp);
861 if (InvalidTemp) {
3
Assuming 'InvalidTemp' is not equal to 0
4
Taking true branch
862 *Invalid = true;
5
Dereference of null pointer (loaded from variable 'Invalid')
863 return StringRef();
864 }
865
866 const char *Begin = File.data() + LocInfo.second;
867 return StringRef(Begin, Tok.getLength());
868}
869
870} // end namespace comments
871} // end namespace clang