clang-tools  9.0.0
ClangdUnit.cpp
Go to the documentation of this file.
1 //===--- ClangdUnit.cpp ------------------------------------------*- C++-*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
9 #include "ClangdUnit.h"
10 #include "../clang-tidy/ClangTidyDiagnosticConsumer.h"
11 #include "../clang-tidy/ClangTidyModuleRegistry.h"
12 #include "Compiler.h"
13 #include "Diagnostics.h"
14 #include "Headers.h"
15 #include "IncludeFixer.h"
16 #include "Logger.h"
17 #include "SourceCode.h"
18 #include "Trace.h"
20 #include "index/Index.h"
21 #include "clang/AST/ASTContext.h"
22 #include "clang/Basic/LangOptions.h"
23 #include "clang/Basic/SourceManager.h"
24 #include "clang/Basic/TokenKinds.h"
25 #include "clang/Frontend/CompilerInstance.h"
26 #include "clang/Frontend/CompilerInvocation.h"
27 #include "clang/Frontend/FrontendActions.h"
28 #include "clang/Frontend/Utils.h"
29 #include "clang/Index/IndexDataConsumer.h"
30 #include "clang/Index/IndexingAction.h"
31 #include "clang/Lex/Lexer.h"
32 #include "clang/Lex/MacroInfo.h"
33 #include "clang/Lex/Preprocessor.h"
34 #include "clang/Lex/PreprocessorOptions.h"
35 #include "clang/Sema/Sema.h"
36 #include "clang/Serialization/ASTWriter.h"
37 #include "clang/Serialization/PCHContainerOperations.h"
38 #include "clang/Tooling/CompilationDatabase.h"
39 #include "clang/Tooling/Syntax/Tokens.h"
40 #include "llvm/ADT/ArrayRef.h"
41 #include "llvm/ADT/STLExtras.h"
42 #include "llvm/ADT/SmallString.h"
43 #include "llvm/ADT/SmallVector.h"
44 #include "llvm/Support/raw_ostream.h"
45 #include <algorithm>
46 #include <memory>
47 
48 namespace clang {
49 namespace clangd {
50 namespace {
51 
52 bool compileCommandsAreEqual(const tooling::CompileCommand &LHS,
53  const tooling::CompileCommand &RHS) {
54  // We don't check for Output, it should not matter to clangd.
55  return LHS.Directory == RHS.Directory && LHS.Filename == RHS.Filename &&
56  llvm::makeArrayRef(LHS.CommandLine).equals(RHS.CommandLine);
57 }
58 
59 template <class T> std::size_t getUsedBytes(const std::vector<T> &Vec) {
60  return Vec.capacity() * sizeof(T);
61 }
62 
63 class DeclTrackingASTConsumer : public ASTConsumer {
64 public:
65  DeclTrackingASTConsumer(std::vector<Decl *> &TopLevelDecls)
66  : TopLevelDecls(TopLevelDecls) {}
67 
68  bool HandleTopLevelDecl(DeclGroupRef DG) override {
69  for (Decl *D : DG) {
70  auto &SM = D->getASTContext().getSourceManager();
71  if (!isInsideMainFile(D->getLocation(), SM))
72  continue;
73 
74  // ObjCMethodDecl are not actually top-level decls.
75  if (isa<ObjCMethodDecl>(D))
76  continue;
77 
78  TopLevelDecls.push_back(D);
79  }
80  return true;
81  }
82 
83 private:
84  std::vector<Decl *> &TopLevelDecls;
85 };
86 
87 class ClangdFrontendAction : public SyntaxOnlyAction {
88 public:
89  std::vector<Decl *> takeTopLevelDecls() { return std::move(TopLevelDecls); }
90 
91 protected:
92  std::unique_ptr<ASTConsumer>
93  CreateASTConsumer(CompilerInstance &CI, llvm::StringRef InFile) override {
94  return llvm::make_unique<DeclTrackingASTConsumer>(/*ref*/ TopLevelDecls);
95  }
96 
97 private:
98  std::vector<Decl *> TopLevelDecls;
99 };
100 
101 class CollectMainFileMacros : public PPCallbacks {
102 public:
103  explicit CollectMainFileMacros(const SourceManager &SM,
104  std::vector<std::string> *Out)
105  : SM(SM), Out(Out) {}
106 
107  void FileChanged(SourceLocation Loc, FileChangeReason,
108  SrcMgr::CharacteristicKind, FileID Prev) {
109  InMainFile = SM.isWrittenInMainFile(Loc);
110  }
111 
112  void MacroDefined(const Token &MacroName, const MacroDirective *MD) {
113  if (InMainFile)
114  MainFileMacros.insert(MacroName.getIdentifierInfo()->getName());
115  }
116 
117  void EndOfMainFile() {
118  for (const auto &Entry : MainFileMacros)
119  Out->push_back(Entry.getKey());
120  llvm::sort(*Out);
121  }
122 
123 private:
124  const SourceManager &SM;
125  bool InMainFile = true;
126  llvm::StringSet<> MainFileMacros;
127  std::vector<std::string> *Out;
128 };
129 
130 class CppFilePreambleCallbacks : public PreambleCallbacks {
131 public:
132  CppFilePreambleCallbacks(PathRef File, PreambleParsedCallback ParsedCallback)
133  : File(File), ParsedCallback(ParsedCallback) {
134  }
135 
136  IncludeStructure takeIncludes() { return std::move(Includes); }
137 
138  std::vector<std::string> takeMainFileMacros() {
139  return std::move(MainFileMacros);
140  }
141 
142  CanonicalIncludes takeCanonicalIncludes() { return std::move(CanonIncludes); }
143 
144  void AfterExecute(CompilerInstance &CI) override {
145  if (!ParsedCallback)
146  return;
147  trace::Span Tracer("Running PreambleCallback");
148  ParsedCallback(CI.getASTContext(), CI.getPreprocessorPtr(), CanonIncludes);
149  }
150 
151  void BeforeExecute(CompilerInstance &CI) override {
152  addSystemHeadersMapping(&CanonIncludes, CI.getLangOpts());
153  SourceMgr = &CI.getSourceManager();
154  }
155 
156  std::unique_ptr<PPCallbacks> createPPCallbacks() override {
157  assert(SourceMgr && "SourceMgr must be set at this point");
158  return llvm::make_unique<PPChainedCallbacks>(
159  collectIncludeStructureCallback(*SourceMgr, &Includes),
160  llvm::make_unique<CollectMainFileMacros>(*SourceMgr, &MainFileMacros));
161  }
162 
163  CommentHandler *getCommentHandler() override {
164  IWYUHandler = collectIWYUHeaderMaps(&CanonIncludes);
165  return IWYUHandler.get();
166  }
167 
168 private:
169  PathRef File;
170  PreambleParsedCallback ParsedCallback;
171  IncludeStructure Includes;
172  CanonicalIncludes CanonIncludes;
173  std::vector<std::string> MainFileMacros;
174  std::unique_ptr<CommentHandler> IWYUHandler = nullptr;
175  SourceManager *SourceMgr = nullptr;
176 };
177 
178 // When using a preamble, only preprocessor events outside its bounds are seen.
179 // This is almost what we want: replaying transitive preprocessing wastes time.
180 // However this confuses clang-tidy checks: they don't see any #includes!
181 // So we replay the *non-transitive* #includes that appear in the main-file.
182 // It would be nice to replay other events (macro definitions, ifdefs etc) but
183 // this addresses the most common cases fairly cheaply.
184 class ReplayPreamble : private PPCallbacks {
185 public:
186  // Attach preprocessor hooks such that preamble events will be injected at
187  // the appropriate time.
188  // Events will be delivered to the *currently registered* PP callbacks.
189  static void attach(const IncludeStructure &Includes,
190  CompilerInstance &Clang) {
191  auto &PP = Clang.getPreprocessor();
192  auto *ExistingCallbacks = PP.getPPCallbacks();
193  // No need to replay events if nobody is listening.
194  if (!ExistingCallbacks)
195  return;
196  PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(
197  new ReplayPreamble(Includes, ExistingCallbacks,
198  Clang.getSourceManager(), PP, Clang.getLangOpts())));
199  // We're relying on the fact that addPPCallbacks keeps the old PPCallbacks
200  // around, creating a chaining wrapper. Guard against other implementations.
201  assert(PP.getPPCallbacks() != ExistingCallbacks &&
202  "Expected chaining implementation");
203  }
204 
205 private:
206  ReplayPreamble(const IncludeStructure &Includes, PPCallbacks *Delegate,
207  const SourceManager &SM, Preprocessor &PP,
208  const LangOptions &LangOpts)
209  : Includes(Includes), Delegate(Delegate), SM(SM), PP(PP),
210  LangOpts(LangOpts) {}
211 
212  // In a normal compile, the preamble traverses the following structure:
213  //
214  // mainfile.cpp
215  // <built-in>
216  // ... macro definitions like __cplusplus ...
217  // <command-line>
218  // ... macro definitions for args like -Dfoo=bar ...
219  // "header1.h"
220  // ... header file contents ...
221  // "header2.h"
222  // ... header file contents ...
223  // ... main file contents ...
224  //
225  // When using a preamble, the "header1" and "header2" subtrees get skipped.
226  // We insert them right after the built-in header, which still appears.
227  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
228  SrcMgr::CharacteristicKind Kind, FileID PrevFID) override {
229  // It'd be nice if there was a better way to identify built-in headers...
230  if (Reason == FileChangeReason::ExitFile &&
231  SM.getBuffer(PrevFID)->getBufferIdentifier() == "<built-in>")
232  replay();
233  }
234 
235  void replay() {
236  for (const auto &Inc : Includes.MainFileIncludes) {
237  const FileEntry *File = nullptr;
238  if (Inc.Resolved != "")
239  File = SM.getFileManager().getFile(Inc.Resolved);
240 
241  llvm::StringRef WrittenFilename =
242  llvm::StringRef(Inc.Written).drop_front().drop_back();
243  bool Angled = llvm::StringRef(Inc.Written).startswith("<");
244 
245  // Re-lex the #include directive to find its interesting parts.
246  llvm::StringRef Src = SM.getBufferData(SM.getMainFileID());
247  Lexer RawLexer(SM.getLocForStartOfFile(SM.getMainFileID()), LangOpts,
248  Src.begin(), Src.begin() + Inc.HashOffset, Src.end());
249  Token HashTok, IncludeTok, FilenameTok;
250  RawLexer.LexFromRawLexer(HashTok);
251  assert(HashTok.getKind() == tok::hash);
252  RawLexer.setParsingPreprocessorDirective(true);
253  RawLexer.LexFromRawLexer(IncludeTok);
254  IdentifierInfo *II = PP.getIdentifierInfo(IncludeTok.getRawIdentifier());
255  IncludeTok.setIdentifierInfo(II);
256  IncludeTok.setKind(II->getTokenID());
257  RawLexer.LexIncludeFilename(FilenameTok);
258 
259  Delegate->InclusionDirective(
260  HashTok.getLocation(), IncludeTok, WrittenFilename, Angled,
261  CharSourceRange::getCharRange(FilenameTok.getLocation(),
262  FilenameTok.getEndLoc()),
263  File, "SearchPath", "RelPath", /*Imported=*/nullptr, Inc.FileKind);
264  if (File)
265  Delegate->FileSkipped(*File, FilenameTok, Inc.FileKind);
266  else {
267  llvm::SmallString<1> UnusedRecovery;
268  Delegate->FileNotFound(WrittenFilename, UnusedRecovery);
269  }
270  }
271  }
272 
273  const IncludeStructure &Includes;
274  PPCallbacks *Delegate;
275  const SourceManager &SM;
276  Preprocessor &PP;
277  const LangOptions &LangOpts;
278 };
279 
280 } // namespace
281 
282 void dumpAST(ParsedAST &AST, llvm::raw_ostream &OS) {
283  AST.getASTContext().getTranslationUnitDecl()->dump(OS, true);
284 }
285 
286 llvm::Optional<ParsedAST>
287 ParsedAST::build(std::unique_ptr<CompilerInvocation> CI,
288  std::shared_ptr<const PreambleData> Preamble,
289  std::unique_ptr<llvm::MemoryBuffer> Buffer,
290  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> VFS,
291  const SymbolIndex *Index, const ParseOptions &Opts) {
292  assert(CI);
293  // Command-line parsing sets DisableFree to true by default, but we don't want
294  // to leak memory in clangd.
295  CI->getFrontendOpts().DisableFree = false;
296  const PrecompiledPreamble *PreamblePCH =
297  Preamble ? &Preamble->Preamble : nullptr;
298 
299  StoreDiags ASTDiags;
300  std::string Content = Buffer->getBuffer();
301 
302  auto Clang = prepareCompilerInstance(std::move(CI), PreamblePCH,
303  std::move(Buffer), VFS, ASTDiags);
304  if (!Clang)
305  return None;
306 
307  auto Action = llvm::make_unique<ClangdFrontendAction>();
308  const FrontendInputFile &MainInput = Clang->getFrontendOpts().Inputs[0];
309  if (!Action->BeginSourceFile(*Clang, MainInput)) {
310  log("BeginSourceFile() failed when building AST for {0}",
311  MainInput.getFile());
312  return None;
313  }
314 
315  // Set up ClangTidy. Must happen after BeginSourceFile() so ASTContext exists.
316  // Clang-tidy has some limitiations to ensure reasonable performance:
317  // - checks don't see all preprocessor events in the preamble
318  // - matchers run only over the main-file top-level decls (and can't see
319  // ancestors outside this scope).
320  // In practice almost all checks work well without modifications.
321  std::vector<std::unique_ptr<tidy::ClangTidyCheck>> CTChecks;
322  ast_matchers::MatchFinder CTFinder;
323  llvm::Optional<tidy::ClangTidyContext> CTContext;
324  {
325  trace::Span Tracer("ClangTidyInit");
326  dlog("ClangTidy configuration for file {0}: {1}", MainInput.getFile(),
328  tidy::ClangTidyCheckFactories CTFactories;
329  for (const auto &E : tidy::ClangTidyModuleRegistry::entries())
330  E.instantiate()->addCheckFactories(CTFactories);
331  CTContext.emplace(llvm::make_unique<tidy::DefaultOptionsProvider>(
333  CTContext->setDiagnosticsEngine(&Clang->getDiagnostics());
334  CTContext->setASTContext(&Clang->getASTContext());
335  CTContext->setCurrentFile(MainInput.getFile());
336  CTFactories.createChecks(CTContext.getPointer(), CTChecks);
337  ASTDiags.setLevelAdjuster([&CTContext](DiagnosticsEngine::Level DiagLevel,
338  const clang::Diagnostic &Info) {
339  if (CTContext) {
340  std::string CheckName = CTContext->getCheckName(Info.getID());
341  bool IsClangTidyDiag = !CheckName.empty();
342  if (IsClangTidyDiag) {
343  // Check for warning-as-error.
344  // We deliberately let this take precedence over suppression comments
345  // to match clang-tidy's behaviour.
346  if (DiagLevel == DiagnosticsEngine::Warning &&
347  CTContext->treatAsError(CheckName)) {
349  }
350 
351  // Check for suppression comment. Skip the check for diagnostics not
352  // in the main file, because we don't want that function to query the
353  // source buffer for preamble files. For the same reason, we ask
354  // ShouldSuppressDiagnostic not to follow macro expansions, since
355  // those might take us into a preamble file as well.
356  bool IsInsideMainFile =
357  Info.hasSourceManager() &&
358  isInsideMainFile(Info.getLocation(), Info.getSourceManager());
359  if (IsInsideMainFile && tidy::ShouldSuppressDiagnostic(
360  DiagLevel, Info, *CTContext,
361  /* CheckMacroExpansion = */ false)) {
362  return DiagnosticsEngine::Ignored;
363  }
364  }
365  }
366  return DiagLevel;
367  });
368  Preprocessor *PP = &Clang->getPreprocessor();
369  for (const auto &Check : CTChecks) {
370  // FIXME: the PP callbacks skip the entire preamble.
371  // Checks that want to see #includes in the main file do not see them.
372  Check->registerPPCallbacks(Clang->getSourceManager(), PP, PP);
373  Check->registerMatchers(&CTFinder);
374  }
375  }
376 
377  // Add IncludeFixer which can recorver diagnostics caused by missing includes
378  // (e.g. incomplete type) and attach include insertion fixes to diagnostics.
379  llvm::Optional<IncludeFixer> FixIncludes;
380  auto BuildDir = VFS->getCurrentWorkingDirectory();
381  if (Opts.SuggestMissingIncludes && Index && !BuildDir.getError()) {
382  auto Style = getFormatStyleForFile(MainInput.getFile(), Content, VFS.get());
383  auto Inserter = std::make_shared<IncludeInserter>(
384  MainInput.getFile(), Content, Style, BuildDir.get(),
385  &Clang->getPreprocessor().getHeaderSearchInfo());
386  if (Preamble) {
387  for (const auto &Inc : Preamble->Includes.MainFileIncludes)
388  Inserter->addExisting(Inc);
389  }
390  FixIncludes.emplace(MainInput.getFile(), Inserter, *Index,
391  /*IndexRequestLimit=*/5);
392  ASTDiags.contributeFixes([&FixIncludes](DiagnosticsEngine::Level DiagLevl,
393  const clang::Diagnostic &Info) {
394  return FixIncludes->fix(DiagLevl, Info);
395  });
396  Clang->setExternalSemaSource(FixIncludes->unresolvedNameRecorder());
397  }
398 
399  // Copy over the includes from the preamble, then combine with the
400  // non-preamble includes below.
401  auto Includes = Preamble ? Preamble->Includes : IncludeStructure{};
402  // Replay the preamble includes so that clang-tidy checks can see them.
403  if (Preamble)
404  ReplayPreamble::attach(Includes, *Clang);
405  // Important: collectIncludeStructure is registered *after* ReplayPreamble!
406  // Otherwise we would collect the replayed includes again...
407  // (We can't *just* use the replayed includes, they don't have Resolved path).
408  Clang->getPreprocessor().addPPCallbacks(
409  collectIncludeStructureCallback(Clang->getSourceManager(), &Includes));
410 
411  // Copy over the includes from the preamble, then combine with the
412  // non-preamble includes below.
413  CanonicalIncludes CanonIncludes;
414  if (Preamble)
415  CanonIncludes = Preamble->CanonIncludes;
416  else
417  addSystemHeadersMapping(&CanonIncludes, Clang->getLangOpts());
418  std::unique_ptr<CommentHandler> IWYUHandler =
419  collectIWYUHeaderMaps(&CanonIncludes);
420  Clang->getPreprocessor().addCommentHandler(IWYUHandler.get());
421 
422  // Collect tokens of the main file.
423  syntax::TokenCollector CollectTokens(Clang->getPreprocessor());
424 
425  if (llvm::Error Err = Action->Execute())
426  log("Execute() failed when building AST for {0}: {1}", MainInput.getFile(),
427  toString(std::move(Err)));
428 
429  // We have to consume the tokens before running clang-tidy to avoid collecting
430  // tokens from running the preprocessor inside the checks (only
431  // modernize-use-trailing-return-type does that today).
432  syntax::TokenBuffer Tokens = std::move(CollectTokens).consume();
433  std::vector<Decl *> ParsedDecls = Action->takeTopLevelDecls();
434  // AST traversals should exclude the preamble, to avoid performance cliffs.
435  Clang->getASTContext().setTraversalScope(ParsedDecls);
436  {
437  // Run the AST-dependent part of the clang-tidy checks.
438  // (The preprocessor part ran already, via PPCallbacks).
439  trace::Span Tracer("ClangTidyMatch");
440  CTFinder.matchAST(Clang->getASTContext());
441  }
442 
443  // UnitDiagsConsumer is local, we can not store it in CompilerInstance that
444  // has a longer lifetime.
445  Clang->getDiagnostics().setClient(new IgnoreDiagnostics);
446  // CompilerInstance won't run this callback, do it directly.
447  ASTDiags.EndSourceFile();
448  // XXX: This is messy: clang-tidy checks flush some diagnostics at EOF.
449  // However Action->EndSourceFile() would destroy the ASTContext!
450  // So just inform the preprocessor of EOF, while keeping everything alive.
451  Clang->getPreprocessor().EndSourceFile();
452 
453  std::vector<Diag> Diags = ASTDiags.take(CTContext.getPointer());
454  // Add diagnostics from the preamble, if any.
455  if (Preamble)
456  Diags.insert(Diags.begin(), Preamble->Diags.begin(), Preamble->Diags.end());
457  return ParsedAST(std::move(Preamble), std::move(Clang), std::move(Action),
458  std::move(Tokens), std::move(ParsedDecls), std::move(Diags),
459  std::move(Includes), std::move(CanonIncludes));
460 }
461 
462 ParsedAST::ParsedAST(ParsedAST &&Other) = default;
463 
464 ParsedAST &ParsedAST::operator=(ParsedAST &&Other) = default;
465 
467  if (Action) {
468  // We already notified the PP of end-of-file earlier, so detach it first.
469  // We must keep it alive until after EndSourceFile(), Sema relies on this.
470  auto PP = Clang->getPreprocessorPtr(); // Keep PP alive for now.
471  Clang->setPreprocessor(nullptr); // Detach so we don't send EOF again.
472  Action->EndSourceFile(); // Destroy ASTContext and Sema.
473  // Now Sema is gone, it's safe for PP to go out of scope.
474  }
475 }
476 
477 ASTContext &ParsedAST::getASTContext() { return Clang->getASTContext(); }
478 
479 const ASTContext &ParsedAST::getASTContext() const {
480  return Clang->getASTContext();
481 }
482 
483 Preprocessor &ParsedAST::getPreprocessor() { return Clang->getPreprocessor(); }
484 
485 std::shared_ptr<Preprocessor> ParsedAST::getPreprocessorPtr() {
486  return Clang->getPreprocessorPtr();
487 }
488 
489 const Preprocessor &ParsedAST::getPreprocessor() const {
490  return Clang->getPreprocessor();
491 }
492 
493 llvm::ArrayRef<Decl *> ParsedAST::getLocalTopLevelDecls() {
494  return LocalTopLevelDecls;
495 }
496 
497 const std::vector<Diag> &ParsedAST::getDiagnostics() const { return Diags; }
498 
499 std::size_t ParsedAST::getUsedBytes() const {
500  auto &AST = getASTContext();
501  // FIXME(ibiryukov): we do not account for the dynamically allocated part of
502  // Message and Fixes inside each diagnostic.
503  std::size_t Total =
504  clangd::getUsedBytes(LocalTopLevelDecls) + clangd::getUsedBytes(Diags);
505 
506  // FIXME: the rest of the function is almost a direct copy-paste from
507  // libclang's clang_getCXTUResourceUsage. We could share the implementation.
508 
509  // Sum up variaous allocators inside the ast context and the preprocessor.
510  Total += AST.getASTAllocatedMemory();
511  Total += AST.getSideTableAllocatedMemory();
512  Total += AST.Idents.getAllocator().getTotalMemory();
513  Total += AST.Selectors.getTotalMemory();
514 
515  Total += AST.getSourceManager().getContentCacheSize();
516  Total += AST.getSourceManager().getDataStructureSizes();
517  Total += AST.getSourceManager().getMemoryBufferSizes().malloc_bytes;
518 
519  if (ExternalASTSource *Ext = AST.getExternalSource())
520  Total += Ext->getMemoryBufferSizes().malloc_bytes;
521 
522  const Preprocessor &PP = getPreprocessor();
523  Total += PP.getTotalMemory();
524  if (PreprocessingRecord *PRec = PP.getPreprocessingRecord())
525  Total += PRec->getTotalMemory();
526  Total += PP.getHeaderSearchInfo().getTotalMemory();
527 
528  return Total;
529 }
530 
532  return Includes;
533 }
534 
536  return CanonIncludes;
537 }
538 
540  std::vector<Diag> Diags, IncludeStructure Includes,
541  std::vector<std::string> MainFileMacros,
542  std::unique_ptr<PreambleFileStatusCache> StatCache,
543  CanonicalIncludes CanonIncludes)
544  : Preamble(std::move(Preamble)), Diags(std::move(Diags)),
545  Includes(std::move(Includes)), MainFileMacros(std::move(MainFileMacros)),
546  StatCache(std::move(StatCache)), CanonIncludes(std::move(CanonIncludes)) {
547 }
548 
549 ParsedAST::ParsedAST(std::shared_ptr<const PreambleData> Preamble,
550  std::unique_ptr<CompilerInstance> Clang,
551  std::unique_ptr<FrontendAction> Action,
552  syntax::TokenBuffer Tokens,
553  std::vector<Decl *> LocalTopLevelDecls,
554  std::vector<Diag> Diags, IncludeStructure Includes,
556  : Preamble(std::move(Preamble)), Clang(std::move(Clang)),
557  Action(std::move(Action)), Tokens(std::move(Tokens)),
558  Diags(std::move(Diags)),
559  LocalTopLevelDecls(std::move(LocalTopLevelDecls)),
560  Includes(std::move(Includes)), CanonIncludes(std::move(CanonIncludes)) {
561  assert(this->Clang);
562  assert(this->Action);
563 }
564 
565 std::shared_ptr<const PreambleData>
566 buildPreamble(PathRef FileName, CompilerInvocation &CI,
567  std::shared_ptr<const PreambleData> OldPreamble,
568  const tooling::CompileCommand &OldCompileCommand,
569  const ParseInputs &Inputs, bool StoreInMemory,
570  PreambleParsedCallback PreambleCallback) {
571  // Note that we don't need to copy the input contents, preamble can live
572  // without those.
573  auto ContentsBuffer = llvm::MemoryBuffer::getMemBuffer(Inputs.Contents);
574  auto Bounds =
575  ComputePreambleBounds(*CI.getLangOpts(), ContentsBuffer.get(), 0);
576 
577  if (OldPreamble &&
578  compileCommandsAreEqual(Inputs.CompileCommand, OldCompileCommand) &&
579  OldPreamble->Preamble.CanReuse(CI, ContentsBuffer.get(), Bounds,
580  Inputs.FS.get())) {
581  vlog("Reusing preamble for file {0}", llvm::Twine(FileName));
582  return OldPreamble;
583  }
584  vlog("Preamble for file {0} cannot be reused. Attempting to rebuild it.",
585  FileName);
586 
587  trace::Span Tracer("BuildPreamble");
588  SPAN_ATTACH(Tracer, "File", FileName);
589  StoreDiags PreambleDiagnostics;
590  llvm::IntrusiveRefCntPtr<DiagnosticsEngine> PreambleDiagsEngine =
591  CompilerInstance::createDiagnostics(&CI.getDiagnosticOpts(),
592  &PreambleDiagnostics, false);
593 
594  // Skip function bodies when building the preamble to speed up building
595  // the preamble and make it smaller.
596  assert(!CI.getFrontendOpts().SkipFunctionBodies);
597  CI.getFrontendOpts().SkipFunctionBodies = true;
598  // We don't want to write comment locations into PCH. They are racy and slow
599  // to read back. We rely on dynamic index for the comments instead.
600  CI.getPreprocessorOpts().WriteCommentListToPCH = false;
601 
602  CppFilePreambleCallbacks SerializedDeclsCollector(FileName, PreambleCallback);
603  if (Inputs.FS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {
604  log("Couldn't set working directory when building the preamble.");
605  // We proceed anyway, our lit-tests rely on results for non-existing working
606  // dirs.
607  }
608 
609  llvm::SmallString<32> AbsFileName(FileName);
610  Inputs.FS->makeAbsolute(AbsFileName);
611  auto StatCache = llvm::make_unique<PreambleFileStatusCache>(AbsFileName);
612  auto BuiltPreamble = PrecompiledPreamble::Build(
613  CI, ContentsBuffer.get(), Bounds, *PreambleDiagsEngine,
614  StatCache->getProducingFS(Inputs.FS),
615  std::make_shared<PCHContainerOperations>(), StoreInMemory,
616  SerializedDeclsCollector);
617 
618  // When building the AST for the main file, we do want the function
619  // bodies.
620  CI.getFrontendOpts().SkipFunctionBodies = false;
621 
622  if (BuiltPreamble) {
623  vlog("Built preamble of size {0} for file {1}", BuiltPreamble->getSize(),
624  FileName);
625  std::vector<Diag> Diags = PreambleDiagnostics.take();
626  return std::make_shared<PreambleData>(
627  std::move(*BuiltPreamble), std::move(Diags),
628  SerializedDeclsCollector.takeIncludes(),
629  SerializedDeclsCollector.takeMainFileMacros(), std::move(StatCache),
630  SerializedDeclsCollector.takeCanonicalIncludes());
631  } else {
632  elog("Could not build a preamble for file {0}", FileName);
633  return nullptr;
634  }
635 }
636 
637 llvm::Optional<ParsedAST>
638 buildAST(PathRef FileName, std::unique_ptr<CompilerInvocation> Invocation,
639  const ParseInputs &Inputs,
640  std::shared_ptr<const PreambleData> Preamble) {
641  trace::Span Tracer("BuildAST");
642  SPAN_ATTACH(Tracer, "File", FileName);
643 
644  auto VFS = Inputs.FS;
645  if (Preamble && Preamble->StatCache)
646  VFS = Preamble->StatCache->getConsumingFS(std::move(VFS));
647  if (VFS->setCurrentWorkingDirectory(Inputs.CompileCommand.Directory)) {
648  log("Couldn't set working directory when building the preamble.");
649  // We proceed anyway, our lit-tests rely on results for non-existing working
650  // dirs.
651  }
652 
653  return ParsedAST::build(llvm::make_unique<CompilerInvocation>(*Invocation),
654  Preamble,
655  llvm::MemoryBuffer::getMemBufferCopy(Inputs.Contents),
656  std::move(VFS), Inputs.Index, Inputs.Opts);
657 }
658 
660  const Position &Pos, const FileID FID) {
661  const ASTContext &AST = Unit.getASTContext();
662  const SourceManager &SourceMgr = AST.getSourceManager();
663  auto Offset = positionToOffset(SourceMgr.getBufferData(FID), Pos);
664  if (!Offset) {
665  log("getBeginningOfIdentifier: {0}", Offset.takeError());
666  return SourceLocation();
667  }
668 
669  // GetBeginningOfToken(pos) is almost what we want, but does the wrong thing
670  // if the cursor is at the end of the identifier.
671  // Instead, we lex at GetBeginningOfToken(pos - 1). The cases are:
672  // 1) at the beginning of an identifier, we'll be looking at something
673  // that isn't an identifier.
674  // 2) at the middle or end of an identifier, we get the identifier.
675  // 3) anywhere outside an identifier, we'll get some non-identifier thing.
676  // We can't actually distinguish cases 1 and 3, but returning the original
677  // location is correct for both!
678  SourceLocation InputLoc = SourceMgr.getComposedLoc(FID, *Offset);
679  if (*Offset == 0) // Case 1 or 3.
680  return SourceMgr.getMacroArgExpandedLocation(InputLoc);
681  SourceLocation Before = SourceMgr.getComposedLoc(FID, *Offset - 1);
682 
683  Before = Lexer::GetBeginningOfToken(Before, SourceMgr, AST.getLangOpts());
684  Token Tok;
685  if (Before.isValid() &&
686  !Lexer::getRawToken(Before, Tok, SourceMgr, AST.getLangOpts(), false) &&
687  Tok.is(tok::raw_identifier))
688  return SourceMgr.getMacroArgExpandedLocation(Before); // Case 2.
689  return SourceMgr.getMacroArgExpandedLocation(InputLoc); // Case 1 or 3.
690 }
691 
692 } // namespace clangd
693 namespace tidy {
694 // Force the linker to link in Clang-tidy modules.
695 #define LINK_TIDY_MODULE(X) \
696  extern volatile int X##ModuleAnchorSource; \
697  static int LLVM_ATTRIBUTE_UNUSED X##ModuleAnchorDestination = \
698  X##ModuleAnchorSource
699 LINK_TIDY_MODULE(CERT);
700 LINK_TIDY_MODULE(Abseil);
701 LINK_TIDY_MODULE(Boost);
702 LINK_TIDY_MODULE(Bugprone);
703 LINK_TIDY_MODULE(LLVM);
704 LINK_TIDY_MODULE(CppCoreGuidelines);
705 LINK_TIDY_MODULE(Fuchsia);
706 LINK_TIDY_MODULE(Google);
707 LINK_TIDY_MODULE(Android);
708 LINK_TIDY_MODULE(Misc);
709 LINK_TIDY_MODULE(Modernize);
710 LINK_TIDY_MODULE(Performance);
711 LINK_TIDY_MODULE(Portability);
712 LINK_TIDY_MODULE(Readability);
713 LINK_TIDY_MODULE(ObjC);
714 LINK_TIDY_MODULE(HICPP);
715 LINK_TIDY_MODULE(Zircon);
716 #undef LINK_TIDY_MODULE
717 } // namespace tidy
718 } // namespace clang
std::unique_ptr< CommentHandler > collectIWYUHeaderMaps(CanonicalIncludes *Includes)
Returns a CommentHandler that parses pragma comment on include files to determine when we should incl...
SourceLocation Loc
&#39;#&#39; location in the include directive
bool ShouldSuppressDiagnostic(DiagnosticsEngine::Level DiagLevel, const Diagnostic &Info, ClangTidyContext &Context, bool CheckMacroExpansion)
Check whether a given diagnostic should be suppressed due to the presence of a "NOLINT" suppression c...
StoreDiags collects the diagnostics that can later be reported by clangd.
Definition: Diagnostics.h:119
ParsedAST & operator=(ParsedAST &&Other)
Preprocessor & getPreprocessor()
Definition: ClangdUnit.cpp:483
llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS
void dumpAST(ParsedAST &AST, llvm::raw_ostream &OS)
For testing/debugging purposes.
Definition: ClangdUnit.cpp:282
static llvm::Optional< ParsedAST > build(std::unique_ptr< clang::CompilerInvocation > CI, std::shared_ptr< const PreambleData > Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, const SymbolIndex *Index, const ParseOptions &Opts)
Attempts to run Clang and store parsed AST.
Definition: ClangdUnit.cpp:287
void contributeFixes(DiagFixer Fixer)
If set, possibly adds fixes for diagnostics using Fixer.
Definition: Diagnostics.h:134
Interface for symbol indexes that can be used for searching or matching symbols among a set of symbol...
Definition: Index.h:85
bool isInsideMainFile(SourceLocation Loc, const SourceManager &SM)
Returns true iff Loc is inside the main file.
Definition: SourceCode.cpp:372
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Definition: Path.h:23
ArrayRef< Decl * > getLocalTopLevelDecls()
This function returns top-level decls present in the main file of the AST.
Definition: ClangdUnit.cpp:493
void addSystemHeadersMapping(CanonicalIncludes *Includes, const LangOptions &Language)
Adds mapping for system headers and some special symbols (e.g.
std::unique_ptr< CompilerInstance > prepareCompilerInstance(std::unique_ptr< clang::CompilerInvocation > CI, const PrecompiledPreamble *Preamble, std::unique_ptr< llvm::MemoryBuffer > Buffer, llvm::IntrusiveRefCntPtr< llvm::vfs::FileSystem > VFS, DiagnosticConsumer &DiagsClient)
Definition: Compiler.cpp:72
Documents should not be synced at all.
tidy::ClangTidyOptions ClangTidyOpts
Definition: Compiler.h:39
A collection of ClangTidyCheckFactory instances.
void vlog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:67
void elog(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:56
ASTContext & getASTContext()
Note that the returned ast will not contain decls from the preamble that were not deserialized during...
Definition: ClangdUnit.cpp:477
llvm::StringRef Src
std::size_t getUsedBytes() const
Returns the esitmated size of the AST and the accessory structures, in bytes.
Definition: ClangdUnit.cpp:499
SourceLocation getBeginningOfIdentifier(const ParsedAST &Unit, const Position &Pos, const FileID FID)
Get the beginning SourceLocation at a specified Pos.
Definition: ClangdUnit.cpp:659
BindArgumentKind Kind
std::shared_ptr< const PreambleData > buildPreamble(PathRef FileName, CompilerInvocation &CI, std::shared_ptr< const PreambleData > OldPreamble, const tooling::CompileCommand &OldCompileCommand, const ParseInputs &Inputs, bool StoreInMemory, PreambleParsedCallback PreambleCallback)
Rebuild the preamble for the new inputs unless the old one can be reused.
Definition: ClangdUnit.cpp:566
Maps a definition location onto an #include file, based on a set of filename rules.
const CanonicalIncludes & getCanonicalIncludes() const
Definition: ClangdUnit.cpp:535
CanonicalIncludes CanonIncludes
Definition: ClangdUnit.h:69
const IncludeStructure & getIncludeStructure() const
Definition: ClangdUnit.cpp:531
std::unique_ptr< PreambleFileStatusCache > StatCache
Definition: ClangdUnit.h:68
#define LINK_TIDY_MODULE(X)
Definition: ClangdUnit.cpp:695
llvm::Expected< size_t > positionToOffset(llvm::StringRef Code, Position P, bool AllowColumnsBeyondLineLength)
Turn a [line, column] pair into an offset in Code.
Definition: SourceCode.cpp:141
llvm::unique_function< void()> Action
void createChecks(ClangTidyContext *Context, std::vector< std::unique_ptr< ClangTidyCheck >> &Checks)
Create instances of all checks matching CheckRegexString and store them in Checks.
std::string configurationAsText(const ClangTidyOptions &Options)
Serializes configuration to a YAML-encoded string.
void log(const char *Fmt, Ts &&... Vals)
Definition: Logger.h:62
static const char * toString(OffsetEncoding OE)
Definition: Protocol.cpp:1017
std::shared_ptr< Preprocessor > getPreprocessorPtr()
Definition: ClangdUnit.cpp:485
llvm::Optional< ParsedAST > buildAST(PathRef FileName, std::unique_ptr< CompilerInvocation > Invocation, const ParseInputs &Inputs, std::shared_ptr< const PreambleData > Preamble)
Build an AST from provided user inputs.
Definition: ClangdUnit.cpp:638
tooling::CompileCommand CompileCommand
Definition: Compiler.h:45
#define dlog(...)
Definition: Logger.h:72
StringRef Tokens
const Decl * D
Definition: XRefs.cpp:868
PathRef FileName
An information message.
format::FormatStyle getFormatStyleForFile(llvm::StringRef File, llvm::StringRef Content, llvm::vfs::FileSystem *FS)
Choose the clang-format style we should apply to a certain file.
Definition: SourceCode.cpp:530
Stores and provides access to parsed AST.
Definition: ClangdUnit.h:73
std::function< void(ASTContext &, std::shared_ptr< clang::Preprocessor >, const CanonicalIncludes &)> PreambleParsedCallback
Definition: ClangdUnit.h:157
const SymbolIndex * Index
Definition: Compiler.h:49
size_t Offset
Information required to run clang, e.g. to parse AST or do code completion.
Definition: Compiler.h:44
const PreambleData * Preamble
std::unique_ptr< PPCallbacks > collectIncludeStructureCallback(const SourceManager &SM, IncludeStructure *Out)
Returns a PPCallback that visits all inclusions in the main file.
Definition: Headers.cpp:113
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
static GeneratorRegistry::Add< MDGenerator > MD(MDGenerator::Format, "Generator for MD output.")
void EndSourceFile() override
std::vector< Diag > Diags
Definition: ClangdUnit.h:58
IntrusiveRefCntPtr< llvm::vfs::FileSystem > FS
Definition: Compiler.h:46
PrecompiledPreamble Preamble
Definition: ClangdUnit.h:57
IncludeStructure Includes
Definition: ClangdUnit.h:61
void setLevelAdjuster(LevelAdjuster Adjuster)
If set, this allows the client of this class to adjust the level of diagnostics, such as promoting wa...
Definition: Diagnostics.h:138
Records an event whose duration is the lifetime of the Span object.
Definition: Trace.h:82
PreambleData(PrecompiledPreamble Preamble, std::vector< Diag > Diags, IncludeStructure Includes, std::vector< std::string > MainFileMacros, std::unique_ptr< PreambleFileStatusCache > StatCache, CanonicalIncludes CanonIncludes)
Definition: ClangdUnit.cpp:539
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Definition: Trace.h:98
ParsedAST(ParsedAST &&Other)
std::vector< Diag > take(const clang::tidy::ClangTidyContext *Tidy=nullptr)
const std::vector< Diag > & getDiagnostics() const
Definition: ClangdUnit.cpp:497
const SymbolIndex * Index
Definition: Dexp.cpp:84