LLVM  9.0.0svn
TGLexer.cpp
Go to the documentation of this file.
1 //===- TGLexer.cpp - Lexer for TableGen -----------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implement the Lexer for TableGen.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "TGLexer.h"
14 #include "llvm/ADT/StringSwitch.h"
15 #include "llvm/ADT/Twine.h"
16 #include "llvm/Config/config.h" // for strtoull()/strtoll() define
17 #include "llvm/Support/Compiler.h"
19 #include "llvm/Support/SourceMgr.h"
20 #include "llvm/TableGen/Error.h"
21 #include <algorithm>
22 #include <cctype>
23 #include <cerrno>
24 #include <cstdint>
25 #include <cstdio>
26 #include <cstdlib>
27 #include <cstring>
28 
29 using namespace llvm;
30 
31 namespace {
32 // A list of supported preprocessing directives with their
33 // internal token kinds and names.
34 struct {
36  const char *Word;
37 } PreprocessorDirs[] = {
38  { tgtok::Ifdef, "ifdef" },
39  { tgtok::Else, "else" },
40  { tgtok::Endif, "endif" },
41  { tgtok::Define, "define" }
42 };
43 } // end anonymous namespace
44 
46  CurBuffer = SrcMgr.getMainFileID();
47  CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
48  CurPtr = CurBuf.begin();
49  TokStart = nullptr;
50 
51  // Pretend that we enter the "top-level" include file.
52  PrepIncludeStack.push_back(
53  make_unique<std::vector<PreprocessorControlDesc>>());
54 
55  // Put all macros defined in the command line into the DefinedMacros set.
56  std::for_each(Macros.begin(), Macros.end(),
57  [this](const std::string &MacroName) {
58  DefinedMacros.insert(MacroName);
59  });
60 }
61 
63  return SMLoc::getFromPointer(TokStart);
64 }
65 
66 /// ReturnError - Set the error to the specified string at the specified
67 /// location. This is defined to always return tgtok::Error.
68 tgtok::TokKind TGLexer::ReturnError(SMLoc Loc, const Twine &Msg) {
69  PrintError(Loc, Msg);
70  return tgtok::Error;
71 }
72 
73 tgtok::TokKind TGLexer::ReturnError(const char *Loc, const Twine &Msg) {
74  return ReturnError(SMLoc::getFromPointer(Loc), Msg);
75 }
76 
77 bool TGLexer::processEOF() {
78  SMLoc ParentIncludeLoc = SrcMgr.getParentIncludeLoc(CurBuffer);
79  if (ParentIncludeLoc != SMLoc()) {
80  // If prepExitInclude() detects a problem with the preprocessing
81  // control stack, it will return false. Pretend that we reached
82  // the final EOF and stop lexing more tokens by returning false
83  // to LexToken().
84  if (!prepExitInclude(false))
85  return false;
86 
87  CurBuffer = SrcMgr.FindBufferContainingLoc(ParentIncludeLoc);
88  CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
89  CurPtr = ParentIncludeLoc.getPointer();
90  // Make sure TokStart points into the parent file's buffer.
91  // LexToken() assigns to it before calling getNextChar(),
92  // so it is pointing into the included file now.
93  TokStart = CurPtr;
94  return true;
95  }
96 
97  // Pretend that we exit the "top-level" include file.
98  // Note that in case of an error (e.g. control stack imbalance)
99  // the routine will issue a fatal error.
100  prepExitInclude(true);
101  return false;
102 }
103 
104 int TGLexer::getNextChar() {
105  char CurChar = *CurPtr++;
106  switch (CurChar) {
107  default:
108  return (unsigned char)CurChar;
109  case 0: {
110  // A nul character in the stream is either the end of the current buffer or
111  // a random nul in the file. Disambiguate that here.
112  if (CurPtr-1 != CurBuf.end())
113  return 0; // Just whitespace.
114 
115  // Otherwise, return end of file.
116  --CurPtr; // Another call to lex will return EOF again.
117  return EOF;
118  }
119  case '\n':
120  case '\r':
121  // Handle the newline character by ignoring it and incrementing the line
122  // count. However, be careful about 'dos style' files with \n\r in them.
123  // Only treat a \n\r or \r\n as a single line.
124  if ((*CurPtr == '\n' || (*CurPtr == '\r')) &&
125  *CurPtr != CurChar)
126  ++CurPtr; // Eat the two char newline sequence.
127  return '\n';
128  }
129 }
130 
131 int TGLexer::peekNextChar(int Index) const {
132  return *(CurPtr + Index);
133 }
134 
135 tgtok::TokKind TGLexer::LexToken(bool FileOrLineStart) {
136  TokStart = CurPtr;
137  // This always consumes at least one character.
138  int CurChar = getNextChar();
139 
140  switch (CurChar) {
141  default:
142  // Handle letters: [a-zA-Z_]
143  if (isalpha(CurChar) || CurChar == '_')
144  return LexIdentifier();
145 
146  // Unknown character, emit an error.
147  return ReturnError(TokStart, "Unexpected character");
148  case EOF:
149  // Lex next token, if we just left an include file.
150  // Note that leaving an include file means that the next
151  // symbol is located at the end of 'include "..."'
152  // construct, so LexToken() is called with default
153  // false parameter.
154  if (processEOF())
155  return LexToken();
156 
157  // Return EOF denoting the end of lexing.
158  return tgtok::Eof;
159 
160  case ':': return tgtok::colon;
161  case ';': return tgtok::semi;
162  case '.': return tgtok::period;
163  case ',': return tgtok::comma;
164  case '<': return tgtok::less;
165  case '>': return tgtok::greater;
166  case ']': return tgtok::r_square;
167  case '{': return tgtok::l_brace;
168  case '}': return tgtok::r_brace;
169  case '(': return tgtok::l_paren;
170  case ')': return tgtok::r_paren;
171  case '=': return tgtok::equal;
172  case '?': return tgtok::question;
173  case '#':
174  if (FileOrLineStart) {
175  tgtok::TokKind Kind = prepIsDirective();
176  if (Kind != tgtok::Error)
177  return lexPreprocessor(Kind);
178  }
179 
180  return tgtok::paste;
181 
182  case '\r':
183  PrintFatalError("getNextChar() must never return '\r'");
184  return tgtok::Error;
185 
186  case 0:
187  case ' ':
188  case '\t':
189  // Ignore whitespace.
190  return LexToken(FileOrLineStart);
191  case '\n':
192  // Ignore whitespace, and identify the new line.
193  return LexToken(true);
194  case '/':
195  // If this is the start of a // comment, skip until the end of the line or
196  // the end of the buffer.
197  if (*CurPtr == '/')
198  SkipBCPLComment();
199  else if (*CurPtr == '*') {
200  if (SkipCComment())
201  return tgtok::Error;
202  } else // Otherwise, this is an error.
203  return ReturnError(TokStart, "Unexpected character");
204  return LexToken(FileOrLineStart);
205  case '-': case '+':
206  case '0': case '1': case '2': case '3': case '4': case '5': case '6':
207  case '7': case '8': case '9': {
208  int NextChar = 0;
209  if (isdigit(CurChar)) {
210  // Allow identifiers to start with a number if it is followed by
211  // an identifier. This can happen with paste operations like
212  // foo#8i.
213  int i = 0;
214  do {
215  NextChar = peekNextChar(i++);
216  } while (isdigit(NextChar));
217 
218  if (NextChar == 'x' || NextChar == 'b') {
219  // If this is [0-9]b[01] or [0-9]x[0-9A-fa-f] this is most
220  // likely a number.
221  int NextNextChar = peekNextChar(i);
222  switch (NextNextChar) {
223  default:
224  break;
225  case '0': case '1':
226  if (NextChar == 'b')
227  return LexNumber();
229  case '2': case '3': case '4': case '5':
230  case '6': case '7': case '8': case '9':
231  case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
232  case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
233  if (NextChar == 'x')
234  return LexNumber();
235  break;
236  }
237  }
238  }
239 
240  if (isalpha(NextChar) || NextChar == '_')
241  return LexIdentifier();
242 
243  return LexNumber();
244  }
245  case '"': return LexString();
246  case '$': return LexVarName();
247  case '[': return LexBracket();
248  case '!': return LexExclaim();
249  }
250 }
251 
252 /// LexString - Lex "[^"]*"
253 tgtok::TokKind TGLexer::LexString() {
254  const char *StrStart = CurPtr;
255 
256  CurStrVal = "";
257 
258  while (*CurPtr != '"') {
259  // If we hit the end of the buffer, report an error.
260  if (*CurPtr == 0 && CurPtr == CurBuf.end())
261  return ReturnError(StrStart, "End of file in string literal");
262 
263  if (*CurPtr == '\n' || *CurPtr == '\r')
264  return ReturnError(StrStart, "End of line in string literal");
265 
266  if (*CurPtr != '\\') {
267  CurStrVal += *CurPtr++;
268  continue;
269  }
270 
271  ++CurPtr;
272 
273  switch (*CurPtr) {
274  case '\\': case '\'': case '"':
275  // These turn into their literal character.
276  CurStrVal += *CurPtr++;
277  break;
278  case 't':
279  CurStrVal += '\t';
280  ++CurPtr;
281  break;
282  case 'n':
283  CurStrVal += '\n';
284  ++CurPtr;
285  break;
286 
287  case '\n':
288  case '\r':
289  return ReturnError(CurPtr, "escaped newlines not supported in tblgen");
290 
291  // If we hit the end of the buffer, report an error.
292  case '\0':
293  if (CurPtr == CurBuf.end())
294  return ReturnError(StrStart, "End of file in string literal");
296  default:
297  return ReturnError(CurPtr, "invalid escape in string literal");
298  }
299  }
300 
301  ++CurPtr;
302  return tgtok::StrVal;
303 }
304 
305 tgtok::TokKind TGLexer::LexVarName() {
306  if (!isalpha(CurPtr[0]) && CurPtr[0] != '_')
307  return ReturnError(TokStart, "Invalid variable name");
308 
309  // Otherwise, we're ok, consume the rest of the characters.
310  const char *VarNameStart = CurPtr++;
311 
312  while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
313  ++CurPtr;
314 
315  CurStrVal.assign(VarNameStart, CurPtr);
316  return tgtok::VarName;
317 }
318 
319 tgtok::TokKind TGLexer::LexIdentifier() {
320  // The first letter is [a-zA-Z_].
321  const char *IdentStart = TokStart;
322 
323  // Match the rest of the identifier regex: [0-9a-zA-Z_]*
324  while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
325  ++CurPtr;
326 
327  // Check to see if this identifier is a keyword.
328  StringRef Str(IdentStart, CurPtr-IdentStart);
329 
330  if (Str == "include") {
331  if (LexInclude()) return tgtok::Error;
332  return Lex();
333  }
334 
336  .Case("int", tgtok::Int)
337  .Case("bit", tgtok::Bit)
338  .Case("bits", tgtok::Bits)
339  .Case("string", tgtok::String)
340  .Case("list", tgtok::List)
341  .Case("code", tgtok::Code)
342  .Case("dag", tgtok::Dag)
343  .Case("class", tgtok::Class)
344  .Case("def", tgtok::Def)
345  .Case("foreach", tgtok::Foreach)
346  .Case("defm", tgtok::Defm)
347  .Case("defset", tgtok::Defset)
348  .Case("multiclass", tgtok::MultiClass)
349  .Case("field", tgtok::Field)
350  .Case("let", tgtok::Let)
351  .Case("in", tgtok::In)
352  .Default(tgtok::Id);
353 
354  if (Kind == tgtok::Id)
355  CurStrVal.assign(Str.begin(), Str.end());
356  return Kind;
357 }
358 
359 /// LexInclude - We just read the "include" token. Get the string token that
360 /// comes next and enter the include.
361 bool TGLexer::LexInclude() {
362  // The token after the include must be a string.
363  tgtok::TokKind Tok = LexToken();
364  if (Tok == tgtok::Error) return true;
365  if (Tok != tgtok::StrVal) {
366  PrintError(getLoc(), "Expected filename after include");
367  return true;
368  }
369 
370  // Get the string.
371  std::string Filename = CurStrVal;
372  std::string IncludedFile;
373 
374  CurBuffer = SrcMgr.AddIncludeFile(Filename, SMLoc::getFromPointer(CurPtr),
375  IncludedFile);
376  if (!CurBuffer) {
377  PrintError(getLoc(), "Could not find include file '" + Filename + "'");
378  return true;
379  }
380 
381  DependenciesMapTy::const_iterator Found = Dependencies.find(IncludedFile);
382  if (Found != Dependencies.end()) {
383  PrintError(getLoc(),
384  "File '" + IncludedFile + "' has already been included.");
385  SrcMgr.PrintMessage(Found->second, SourceMgr::DK_Note,
386  "previously included here");
387  return true;
388  }
389  Dependencies.insert(std::make_pair(IncludedFile, getLoc()));
390  // Save the line number and lex buffer of the includer.
391  CurBuf = SrcMgr.getMemoryBuffer(CurBuffer)->getBuffer();
392  CurPtr = CurBuf.begin();
393 
394  PrepIncludeStack.push_back(
395  make_unique<std::vector<PreprocessorControlDesc>>());
396  return false;
397 }
398 
399 void TGLexer::SkipBCPLComment() {
400  ++CurPtr; // skip the second slash.
401  while (true) {
402  switch (*CurPtr) {
403  case '\n':
404  case '\r':
405  return; // Newline is end of comment.
406  case 0:
407  // If this is the end of the buffer, end the comment.
408  if (CurPtr == CurBuf.end())
409  return;
410  break;
411  }
412  // Otherwise, skip the character.
413  ++CurPtr;
414  }
415 }
416 
417 /// SkipCComment - This skips C-style /**/ comments. The only difference from C
418 /// is that we allow nesting.
419 bool TGLexer::SkipCComment() {
420  ++CurPtr; // skip the star.
421  unsigned CommentDepth = 1;
422 
423  while (true) {
424  int CurChar = getNextChar();
425  switch (CurChar) {
426  case EOF:
427  PrintError(TokStart, "Unterminated comment!");
428  return true;
429  case '*':
430  // End of the comment?
431  if (CurPtr[0] != '/') break;
432 
433  ++CurPtr; // End the */.
434  if (--CommentDepth == 0)
435  return false;
436  break;
437  case '/':
438  // Start of a nested comment?
439  if (CurPtr[0] != '*') break;
440  ++CurPtr;
441  ++CommentDepth;
442  break;
443  }
444  }
445 }
446 
447 /// LexNumber - Lex:
448 /// [-+]?[0-9]+
449 /// 0x[0-9a-fA-F]+
450 /// 0b[01]+
451 tgtok::TokKind TGLexer::LexNumber() {
452  if (CurPtr[-1] == '0') {
453  if (CurPtr[0] == 'x') {
454  ++CurPtr;
455  const char *NumStart = CurPtr;
456  while (isxdigit(CurPtr[0]))
457  ++CurPtr;
458 
459  // Requires at least one hex digit.
460  if (CurPtr == NumStart)
461  return ReturnError(TokStart, "Invalid hexadecimal number");
462 
463  errno = 0;
464  CurIntVal = strtoll(NumStart, nullptr, 16);
465  if (errno == EINVAL)
466  return ReturnError(TokStart, "Invalid hexadecimal number");
467  if (errno == ERANGE) {
468  errno = 0;
469  CurIntVal = (int64_t)strtoull(NumStart, nullptr, 16);
470  if (errno == EINVAL)
471  return ReturnError(TokStart, "Invalid hexadecimal number");
472  if (errno == ERANGE)
473  return ReturnError(TokStart, "Hexadecimal number out of range");
474  }
475  return tgtok::IntVal;
476  } else if (CurPtr[0] == 'b') {
477  ++CurPtr;
478  const char *NumStart = CurPtr;
479  while (CurPtr[0] == '0' || CurPtr[0] == '1')
480  ++CurPtr;
481 
482  // Requires at least one binary digit.
483  if (CurPtr == NumStart)
484  return ReturnError(CurPtr-2, "Invalid binary number");
485  CurIntVal = strtoll(NumStart, nullptr, 2);
486  return tgtok::BinaryIntVal;
487  }
488  }
489 
490  // Check for a sign without a digit.
491  if (!isdigit(CurPtr[0])) {
492  if (CurPtr[-1] == '-')
493  return tgtok::minus;
494  else if (CurPtr[-1] == '+')
495  return tgtok::plus;
496  }
497 
498  while (isdigit(CurPtr[0]))
499  ++CurPtr;
500  CurIntVal = strtoll(TokStart, nullptr, 10);
501  return tgtok::IntVal;
502 }
503 
504 /// LexBracket - We just read '['. If this is a code block, return it,
505 /// otherwise return the bracket. Match: '[' and '[{ ( [^}]+ | }[^]] )* }]'
506 tgtok::TokKind TGLexer::LexBracket() {
507  if (CurPtr[0] != '{')
508  return tgtok::l_square;
509  ++CurPtr;
510  const char *CodeStart = CurPtr;
511  while (true) {
512  int Char = getNextChar();
513  if (Char == EOF) break;
514 
515  if (Char != '}') continue;
516 
517  Char = getNextChar();
518  if (Char == EOF) break;
519  if (Char == ']') {
520  CurStrVal.assign(CodeStart, CurPtr-2);
521  return tgtok::CodeFragment;
522  }
523  }
524 
525  return ReturnError(CodeStart-2, "Unterminated Code Block");
526 }
527 
528 /// LexExclaim - Lex '!' and '![a-zA-Z]+'.
529 tgtok::TokKind TGLexer::LexExclaim() {
530  if (!isalpha(*CurPtr))
531  return ReturnError(CurPtr - 1, "Invalid \"!operator\"");
532 
533  const char *Start = CurPtr++;
534  while (isalpha(*CurPtr))
535  ++CurPtr;
536 
537  // Check to see which operator this is.
539  StringSwitch<tgtok::TokKind>(StringRef(Start, CurPtr - Start))
540  .Case("eq", tgtok::XEq)
541  .Case("ne", tgtok::XNe)
542  .Case("le", tgtok::XLe)
543  .Case("lt", tgtok::XLt)
544  .Case("ge", tgtok::XGe)
545  .Case("gt", tgtok::XGt)
546  .Case("if", tgtok::XIf)
547  .Case("cond", tgtok::XCond)
548  .Case("isa", tgtok::XIsA)
549  .Case("head", tgtok::XHead)
550  .Case("tail", tgtok::XTail)
551  .Case("size", tgtok::XSize)
552  .Case("con", tgtok::XConcat)
553  .Case("dag", tgtok::XDag)
554  .Case("add", tgtok::XADD)
555  .Case("mul", tgtok::XMUL)
556  .Case("and", tgtok::XAND)
557  .Case("or", tgtok::XOR)
558  .Case("shl", tgtok::XSHL)
559  .Case("sra", tgtok::XSRA)
560  .Case("srl", tgtok::XSRL)
561  .Case("cast", tgtok::XCast)
562  .Case("empty", tgtok::XEmpty)
563  .Case("subst", tgtok::XSubst)
564  .Case("foldl", tgtok::XFoldl)
565  .Case("foreach", tgtok::XForEach)
566  .Case("listconcat", tgtok::XListConcat)
567  .Case("listsplat", tgtok::XListSplat)
568  .Case("strconcat", tgtok::XStrConcat)
570 
571  return Kind != tgtok::Error ? Kind : ReturnError(Start-1, "Unknown operator");
572 }
573 
574 bool TGLexer::prepExitInclude(bool IncludeStackMustBeEmpty) {
575  // Report an error, if preprocessor control stack for the current
576  // file is not empty.
577  if (!PrepIncludeStack.back()->empty()) {
578  prepReportPreprocessorStackError();
579 
580  return false;
581  }
582 
583  // Pop the preprocessing controls from the include stack.
584  if (PrepIncludeStack.empty()) {
585  PrintFatalError("Preprocessor include stack is empty");
586  }
587 
588  PrepIncludeStack.pop_back();
589 
590  if (IncludeStackMustBeEmpty) {
591  if (!PrepIncludeStack.empty())
592  PrintFatalError("Preprocessor include stack is not empty");
593  } else {
594  if (PrepIncludeStack.empty())
595  PrintFatalError("Preprocessor include stack is empty");
596  }
597 
598  return true;
599 }
600 
601 tgtok::TokKind TGLexer::prepIsDirective() const {
602  for (unsigned ID = 0; ID < llvm::array_lengthof(PreprocessorDirs); ++ID) {
603  int NextChar = *CurPtr;
604  bool Match = true;
605  unsigned I = 0;
606  for (; I < strlen(PreprocessorDirs[ID].Word); ++I) {
607  if (NextChar != PreprocessorDirs[ID].Word[I]) {
608  Match = false;
609  break;
610  }
611 
612  NextChar = peekNextChar(I + 1);
613  }
614 
615  // Check for whitespace after the directive. If there is no whitespace,
616  // then we do not recognize it as a preprocessing directive.
617  if (Match) {
618  tgtok::TokKind Kind = PreprocessorDirs[ID].Kind;
619 
620  // New line and EOF may follow only #else/#endif. It will be reported
621  // as an error for #ifdef/#define after the call to prepLexMacroName().
622  if (NextChar == ' ' || NextChar == '\t' || NextChar == EOF ||
623  NextChar == '\n' ||
624  // It looks like TableGen does not support '\r' as the actual
625  // carriage return, e.g. getNextChar() treats a single '\r'
626  // as '\n'. So we do the same here.
627  NextChar == '\r')
628  return Kind;
629 
630  // Allow comments after some directives, e.g.:
631  // #else// OR #else/**/
632  // #endif// OR #endif/**/
633  //
634  // Note that we do allow comments after #ifdef/#define here, e.g.
635  // #ifdef/**/ AND #ifdef//
636  // #define/**/ AND #define//
637  //
638  // These cases will be reported as incorrect after calling
639  // prepLexMacroName(). We could have supported C-style comments
640  // after #ifdef/#define, but this would complicate the code
641  // for little benefit.
642  if (NextChar == '/') {
643  NextChar = peekNextChar(I + 1);
644 
645  if (NextChar == '*' || NextChar == '/')
646  return Kind;
647 
648  // Pretend that we do not recognize the directive.
649  }
650  }
651  }
652 
653  return tgtok::Error;
654 }
655 
656 bool TGLexer::prepEatPreprocessorDirective(tgtok::TokKind Kind) {
657  TokStart = CurPtr;
658 
659  for (unsigned ID = 0; ID < llvm::array_lengthof(PreprocessorDirs); ++ID)
660  if (PreprocessorDirs[ID].Kind == Kind) {
661  // Advance CurPtr to the end of the preprocessing word.
662  CurPtr += strlen(PreprocessorDirs[ID].Word);
663  return true;
664  }
665 
666  PrintFatalError("Unsupported preprocessing token in "
667  "prepEatPreprocessorDirective()");
668  return false;
669 }
670 
671 tgtok::TokKind TGLexer::lexPreprocessor(
672  tgtok::TokKind Kind, bool ReturnNextLiveToken) {
673 
674  // We must be looking at a preprocessing directive. Eat it!
675  if (!prepEatPreprocessorDirective(Kind))
676  PrintFatalError("lexPreprocessor() called for unknown "
677  "preprocessor directive");
678 
679  if (Kind == tgtok::Ifdef) {
680  StringRef MacroName = prepLexMacroName();
681  if (MacroName.empty())
682  return ReturnError(TokStart, "Expected macro name after #ifdef");
683 
684  bool MacroIsDefined = DefinedMacros.count(MacroName) != 0;
685 
686  // Regardless of whether we are processing tokens or not,
687  // we put the #ifdef control on stack.
688  PrepIncludeStack.back()->push_back(
689  {Kind, MacroIsDefined, SMLoc::getFromPointer(TokStart)});
690 
691  if (!prepSkipDirectiveEnd())
692  return ReturnError(CurPtr,
693  "Only comments are supported after #ifdef NAME");
694 
695  // If we were not processing tokens before this #ifdef,
696  // then just return back to the lines skipping code.
697  if (!ReturnNextLiveToken)
698  return Kind;
699 
700  // If we were processing tokens before this #ifdef,
701  // and the macro is defined, then just return the next token.
702  if (MacroIsDefined)
703  return LexToken();
704 
705  // We were processing tokens before this #ifdef, and the macro
706  // is not defined, so we have to start skipping the lines.
707  // If the skipping is successful, it will return the token following
708  // either #else or #endif corresponding to this #ifdef.
709  if (prepSkipRegion(ReturnNextLiveToken))
710  return LexToken();
711 
712  return tgtok::Error;
713  } else if (Kind == tgtok::Else) {
714  // Check if this #else is correct before calling prepSkipDirectiveEnd(),
715  // which will move CurPtr away from the beginning of #else.
716  if (PrepIncludeStack.back()->empty())
717  return ReturnError(TokStart, "#else without #ifdef");
718 
719  PreprocessorControlDesc IfdefEntry = PrepIncludeStack.back()->back();
720 
721  if (IfdefEntry.Kind != tgtok::Ifdef) {
722  PrintError(TokStart, "double #else");
723  return ReturnError(IfdefEntry.SrcPos, "Previous #else is here");
724  }
725 
726  // Replace the corresponding #ifdef's control with its negation
727  // on the control stack.
728  PrepIncludeStack.back()->pop_back();
729  PrepIncludeStack.back()->push_back(
730  {Kind, !IfdefEntry.IsDefined, SMLoc::getFromPointer(TokStart)});
731 
732  if (!prepSkipDirectiveEnd())
733  return ReturnError(CurPtr, "Only comments are supported after #else");
734 
735  // If we were processing tokens before this #else,
736  // we have to start skipping lines until the matching #endif.
737  if (ReturnNextLiveToken) {
738  if (prepSkipRegion(ReturnNextLiveToken))
739  return LexToken();
740 
741  return tgtok::Error;
742  }
743 
744  // Return to the lines skipping code.
745  return Kind;
746  } else if (Kind == tgtok::Endif) {
747  // Check if this #endif is correct before calling prepSkipDirectiveEnd(),
748  // which will move CurPtr away from the beginning of #endif.
749  if (PrepIncludeStack.back()->empty())
750  return ReturnError(TokStart, "#endif without #ifdef");
751 
752  auto &IfdefOrElseEntry = PrepIncludeStack.back()->back();
753 
754  if (IfdefOrElseEntry.Kind != tgtok::Ifdef &&
755  IfdefOrElseEntry.Kind != tgtok::Else) {
756  PrintFatalError("Invalid preprocessor control on the stack");
757  return tgtok::Error;
758  }
759 
760  if (!prepSkipDirectiveEnd())
761  return ReturnError(CurPtr, "Only comments are supported after #endif");
762 
763  PrepIncludeStack.back()->pop_back();
764 
765  // If we were processing tokens before this #endif, then
766  // we should continue it.
767  if (ReturnNextLiveToken) {
768  return LexToken();
769  }
770 
771  // Return to the lines skipping code.
772  return Kind;
773  } else if (Kind == tgtok::Define) {
774  StringRef MacroName = prepLexMacroName();
775  if (MacroName.empty())
776  return ReturnError(TokStart, "Expected macro name after #define");
777 
778  if (!DefinedMacros.insert(MacroName).second)
780  "Duplicate definition of macro: " + Twine(MacroName));
781 
782  if (!prepSkipDirectiveEnd())
783  return ReturnError(CurPtr,
784  "Only comments are supported after #define NAME");
785 
786  if (!ReturnNextLiveToken) {
787  PrintFatalError("#define must be ignored during the lines skipping");
788  return tgtok::Error;
789  }
790 
791  return LexToken();
792  }
793 
794  PrintFatalError("Preprocessing directive is not supported");
795  return tgtok::Error;
796 }
797 
798 bool TGLexer::prepSkipRegion(bool MustNeverBeFalse) {
799  if (!MustNeverBeFalse)
800  PrintFatalError("Invalid recursion.");
801 
802  do {
803  // Skip all symbols to the line end.
804  prepSkipToLineEnd();
805 
806  // Find the first non-whitespace symbol in the next line(s).
807  if (!prepSkipLineBegin())
808  return false;
809 
810  // If the first non-blank/comment symbol on the line is '#',
811  // it may be a start of preprocessing directive.
812  //
813  // If it is not '#' just go to the next line.
814  if (*CurPtr == '#')
815  ++CurPtr;
816  else
817  continue;
818 
819  tgtok::TokKind Kind = prepIsDirective();
820 
821  // If we did not find a preprocessing directive or it is #define,
822  // then just skip to the next line. We do not have to do anything
823  // for #define in the line-skipping mode.
824  if (Kind == tgtok::Error || Kind == tgtok::Define)
825  continue;
826 
827  tgtok::TokKind ProcessedKind = lexPreprocessor(Kind, false);
828 
829  // If lexPreprocessor() encountered an error during lexing this
830  // preprocessor idiom, then return false to the calling lexPreprocessor().
831  // This will force tgtok::Error to be returned to the tokens processing.
832  if (ProcessedKind == tgtok::Error)
833  return false;
834 
835  if (Kind != ProcessedKind)
836  PrintFatalError("prepIsDirective() and lexPreprocessor() "
837  "returned different token kinds");
838 
839  // If this preprocessing directive enables tokens processing,
840  // then return to the lexPreprocessor() and get to the next token.
841  // We can move from line-skipping mode to processing tokens only
842  // due to #else or #endif.
843  if (prepIsProcessingEnabled()) {
844  if (Kind != tgtok::Else && Kind != tgtok::Endif) {
845  PrintFatalError("Tokens processing was enabled by an unexpected "
846  "preprocessing directive");
847  return false;
848  }
849 
850  return true;
851  }
852  } while (CurPtr != CurBuf.end());
853 
854  // We have reached the end of the file, but never left the lines-skipping
855  // mode. This means there is no matching #endif.
856  prepReportPreprocessorStackError();
857  return false;
858 }
859 
860 StringRef TGLexer::prepLexMacroName() {
861  // Skip whitespaces between the preprocessing directive and the macro name.
862  while (*CurPtr == ' ' || *CurPtr == '\t')
863  ++CurPtr;
864 
865  TokStart = CurPtr;
866  // Macro names start with [a-zA-Z_].
867  if (*CurPtr != '_' && !isalpha(*CurPtr))
868  return "";
869 
870  // Match the rest of the identifier regex: [0-9a-zA-Z_]*
871  while (isalpha(*CurPtr) || isdigit(*CurPtr) || *CurPtr == '_')
872  ++CurPtr;
873 
874  return StringRef(TokStart, CurPtr - TokStart);
875 }
876 
877 bool TGLexer::prepSkipLineBegin() {
878  while (CurPtr != CurBuf.end()) {
879  switch (*CurPtr) {
880  case ' ':
881  case '\t':
882  case '\n':
883  case '\r':
884  break;
885 
886  case '/': {
887  int NextChar = peekNextChar(1);
888  if (NextChar == '*') {
889  // Skip C-style comment.
890  // Note that we do not care about skipping the C++-style comments.
891  // If the line contains "//", it may not contain any processable
892  // preprocessing directive. Just return CurPtr pointing to
893  // the first '/' in this case. We also do not care about
894  // incorrect symbols after the first '/' - we are in lines-skipping
895  // mode, so incorrect code is allowed to some extent.
896 
897  // Set TokStart to the beginning of the comment to enable proper
898  // diagnostic printing in case of error in SkipCComment().
899  TokStart = CurPtr;
900 
901  // CurPtr must point to '*' before call to SkipCComment().
902  ++CurPtr;
903  if (SkipCComment())
904  return false;
905  } else {
906  // CurPtr points to the non-whitespace '/'.
907  return true;
908  }
909 
910  // We must not increment CurPtr after the comment was lexed.
911  continue;
912  }
913 
914  default:
915  return true;
916  }
917 
918  ++CurPtr;
919  }
920 
921  // We have reached the end of the file. Return to the lines skipping
922  // code, and allow it to handle the EOF as needed.
923  return true;
924 }
925 
926 bool TGLexer::prepSkipDirectiveEnd() {
927  while (CurPtr != CurBuf.end()) {
928  switch (*CurPtr) {
929  case ' ':
930  case '\t':
931  break;
932 
933  case '\n':
934  case '\r':
935  return true;
936 
937  case '/': {
938  int NextChar = peekNextChar(1);
939  if (NextChar == '/') {
940  // Skip C++-style comment.
941  // We may just return true now, but let's skip to the line/buffer end
942  // to simplify the method specification.
943  ++CurPtr;
944  SkipBCPLComment();
945  } else if (NextChar == '*') {
946  // When we are skipping C-style comment at the end of a preprocessing
947  // directive, we can skip several lines. If any meaningful TD token
948  // follows the end of the C-style comment on the same line, it will
949  // be considered as an invalid usage of TD token.
950  // For example, we want to forbid usages like this one:
951  // #define MACRO class Class {}
952  // But with C-style comments we also disallow the following:
953  // #define MACRO /* This macro is used
954  // to ... */ class Class {}
955  // One can argue that this should be allowed, but it does not seem
956  // to be worth of the complication. Moreover, this matches
957  // the C preprocessor behavior.
958 
959  // Set TokStart to the beginning of the comment to enable proper
960  // diagnostic printer in case of error in SkipCComment().
961  TokStart = CurPtr;
962  ++CurPtr;
963  if (SkipCComment())
964  return false;
965  } else {
966  TokStart = CurPtr;
967  PrintError(CurPtr, "Unexpected character");
968  return false;
969  }
970 
971  // We must not increment CurPtr after the comment was lexed.
972  continue;
973  }
974 
975  default:
976  // Do not allow any non-whitespaces after the directive.
977  TokStart = CurPtr;
978  return false;
979  }
980 
981  ++CurPtr;
982  }
983 
984  return true;
985 }
986 
987 void TGLexer::prepSkipToLineEnd() {
988  while (*CurPtr != '\n' && *CurPtr != '\r' && CurPtr != CurBuf.end())
989  ++CurPtr;
990 }
991 
992 bool TGLexer::prepIsProcessingEnabled() {
993  for (auto I = PrepIncludeStack.back()->rbegin(),
994  E = PrepIncludeStack.back()->rend();
995  I != E; ++I) {
996  if (!I->IsDefined)
997  return false;
998  }
999 
1000  return true;
1001 }
1002 
1003 void TGLexer::prepReportPreprocessorStackError() {
1004  if (PrepIncludeStack.back()->empty())
1005  PrintFatalError("prepReportPreprocessorStackError() called with "
1006  "empty control stack");
1007 
1008  auto &PrepControl = PrepIncludeStack.back()->back();
1009  PrintError(CurBuf.end(), "Reached EOF without matching #endif");
1010  PrintError(PrepControl.SrcPos, "The latest preprocessor control is here");
1011 
1012  TokStart = CurPtr;
1013 }
SMLoc getLoc() const
Definition: TGLexer.cpp:62
unsigned FindBufferContainingLoc(SMLoc Loc) const
Return the ID of the buffer containing the specified location.
Definition: SourceMgr.cpp:61
StringRef getBuffer() const
Definition: MemoryBuffer.h:63
This class represents lattice values for constants.
Definition: AllocatorList.h:23
iterator begin() const
Definition: ArrayRef.h:136
SourceMgr SrcMgr
Definition: Error.cpp:23
std::enable_if<!std::is_array< T >::value, std::unique_ptr< T > >::type make_unique(Args &&... args)
Constructs a new T() with the given args and returns a unique_ptr<T> which owns the object...
Definition: STLExtras.h:1393
StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:67
void PrintMessage(raw_ostream &OS, SMLoc Loc, DiagKind Kind, const Twine &Msg, ArrayRef< SMRange > Ranges=None, ArrayRef< SMFixIt > FixIts=None, bool ShowColors=true) const
Emit a message about the specified location with the specified string.
Definition: SourceMgr.cpp:247
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
support::ulittle32_t Word
Definition: IRSymtab.h:50
LLVM_NODISCARD R Default(T Value)
Definition: StringSwitch.h:181
LLVM_NODISCARD bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:126
const MemoryBuffer * getMemoryBuffer(unsigned i) const
Definition: SourceMgr.h:130
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:32
unsigned getMainFileID() const
Definition: SourceMgr.h:139
const char * getPointer() const
Definition: SMLoc.h:34
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:42
size_type count(StringRef Key) const
count - Return 1 if the element is in the map, 0 otherwise.
Definition: StringMap.h:358
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
TGLexer(SourceMgr &SrcMgr, ArrayRef< std::string > Macros)
Definition: TGLexer.cpp:45
void PrintWarning(ArrayRef< SMLoc > WarningLoc, const Twine &Msg)
Definition: Error.cpp:46
This owns the files read by a parser, handles include stacks, and handles diagnostic wrangling...
Definition: SourceMgr.h:41
tgtok::TokKind Lex()
Definition: TGLexer.h:98
std::pair< typename base::iterator, bool > insert(StringRef Key)
Definition: StringSet.h:38
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
Definition: STLExtras.h:1043
iterator end() const
Definition: ArrayRef.h:137
LLVM_ATTRIBUTE_NORETURN void PrintFatalError(const Twine &Msg)
Definition: Error.cpp:66
iterator begin() const
Definition: StringRef.h:101
static SMLoc getFromPointer(const char *Ptr)
Definition: SMLoc.h:36
unsigned AddIncludeFile(const std::string &Filename, SMLoc IncludeLoc, std::string &IncludedFile)
Search for a file with the specified name in the current directory or in one of the IncludeDirs...
Definition: SourceMgr.cpp:40
SMLoc getParentIncludeLoc(unsigned i) const
Definition: SourceMgr.h:144
#define I(x, y, z)
Definition: MD5.cpp:58
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:250
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
UnaryPredicate for_each(R &&Range, UnaryPredicate P)
Provide wrappers to std::for_each which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1178
Represents a location in source code.
Definition: SMLoc.h:23
iterator end() const
Definition: StringRef.h:103
void PrintError(ArrayRef< SMLoc > ErrorLoc, const Twine &Msg)
Definition: Error.cpp:56