clang-tools  7.0.0
GlobalSymbolBuilderMain.cpp
Go to the documentation of this file.
1 //===--- GlobalSymbolBuilderMain.cpp -----------------------------*- C++-*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // GlobalSymbolBuilder is a tool to generate YAML-format symbols across the
11 // whole project. This tools is for **experimental** only. Don't use it in
12 // production code.
13 //
14 //===---------------------------------------------------------------------===//
15 
17 #include "index/Index.h"
18 #include "index/Merge.h"
19 #include "index/SymbolCollector.h"
20 #include "index/SymbolYAML.h"
21 #include "clang/Frontend/CompilerInstance.h"
22 #include "clang/Frontend/FrontendActions.h"
23 #include "clang/Index/IndexDataConsumer.h"
24 #include "clang/Index/IndexingAction.h"
25 #include "clang/Tooling/CommonOptionsParser.h"
26 #include "clang/Tooling/Execution.h"
27 #include "clang/Tooling/Tooling.h"
28 #include "llvm/Support/CommandLine.h"
29 #include "llvm/Support/MemoryBuffer.h"
30 #include "llvm/Support/Path.h"
31 #include "llvm/Support/Signals.h"
32 #include "llvm/Support/ThreadPool.h"
33 #include "llvm/Support/YAMLTraits.h"
34 
35 using namespace llvm;
36 using namespace clang::tooling;
38 
39 namespace clang {
40 namespace clangd {
41 namespace {
42 
43 static llvm::cl::opt<std::string> AssumedHeaderDir(
44  "assume-header-dir",
45  llvm::cl::desc("The index includes header that a symbol is defined in. "
46  "If the absolute path cannot be determined (e.g. an "
47  "in-memory VFS) then the relative path is resolved against "
48  "this directory, which must be absolute. If this flag is "
49  "not given, such headers will have relative paths."),
50  llvm::cl::init(""));
51 
52 class SymbolIndexActionFactory : public tooling::FrontendActionFactory {
53 public:
54  SymbolIndexActionFactory(tooling::ExecutionContext *Ctx) : Ctx(Ctx) {}
55 
56  clang::FrontendAction *create() override {
57  // Wraps the index action and reports collected symbols to the execution
58  // context at the end of each translation unit.
59  class WrappedIndexAction : public WrapperFrontendAction {
60  public:
61  WrappedIndexAction(std::shared_ptr<SymbolCollector> C,
62  std::unique_ptr<CanonicalIncludes> Includes,
63  const index::IndexingOptions &Opts,
64  tooling::ExecutionContext *Ctx)
65  : WrapperFrontendAction(
66  index::createIndexingAction(C, Opts, nullptr)),
67  Ctx(Ctx), Collector(C), Includes(std::move(Includes)),
68  PragmaHandler(collectIWYUHeaderMaps(this->Includes.get())) {}
69 
70  std::unique_ptr<ASTConsumer>
71  CreateASTConsumer(CompilerInstance &CI, StringRef InFile) override {
72  CI.getPreprocessor().addCommentHandler(PragmaHandler.get());
73  return WrapperFrontendAction::CreateASTConsumer(CI, InFile);
74  }
75 
76  bool BeginInvocation(CompilerInstance &CI) override {
77  // We want all comments, not just the doxygen ones.
78  CI.getLangOpts().CommentOpts.ParseAllComments = true;
79  return WrapperFrontendAction::BeginInvocation(CI);
80  }
81 
82  void EndSourceFileAction() override {
83  WrapperFrontendAction::EndSourceFileAction();
84 
85  const auto &CI = getCompilerInstance();
86  if (CI.hasDiagnostics() &&
87  CI.getDiagnostics().hasUncompilableErrorOccurred()) {
88  llvm::errs()
89  << "Found uncompilable errors in the translation unit. Igoring "
90  "collected symbols...\n";
91  return;
92  }
93 
94  auto Symbols = Collector->takeSymbols();
95  for (const auto &Sym : Symbols) {
96  Ctx->reportResult(Sym.ID.str(), SymbolToYAML(Sym));
97  }
98  }
99 
100  private:
101  tooling::ExecutionContext *Ctx;
102  std::shared_ptr<SymbolCollector> Collector;
103  std::unique_ptr<CanonicalIncludes> Includes;
104  std::unique_ptr<CommentHandler> PragmaHandler;
105  };
106 
107  index::IndexingOptions IndexOpts;
108  IndexOpts.SystemSymbolFilter =
109  index::IndexingOptions::SystemSymbolFilterKind::All;
110  IndexOpts.IndexFunctionLocals = false;
111  auto CollectorOpts = SymbolCollector::Options();
112  CollectorOpts.FallbackDir = AssumedHeaderDir;
113  CollectorOpts.CollectIncludePath = true;
114  CollectorOpts.CountReferences = true;
115  CollectorOpts.Origin = SymbolOrigin::Static;
116  auto Includes = llvm::make_unique<CanonicalIncludes>();
117  addSystemHeadersMapping(Includes.get());
118  CollectorOpts.Includes = Includes.get();
119  return new WrappedIndexAction(
120  std::make_shared<SymbolCollector>(std::move(CollectorOpts)),
121  std::move(Includes), IndexOpts, Ctx);
122  }
123 
124  tooling::ExecutionContext *Ctx;
125 };
126 
127 // Combine occurrences of the same symbol across translation units.
128 SymbolSlab mergeSymbols(tooling::ToolResults *Results) {
129  SymbolSlab::Builder UniqueSymbols;
130  llvm::BumpPtrAllocator Arena;
131  Symbol::Details Scratch;
132  Results->forEachResult([&](llvm::StringRef Key, llvm::StringRef Value) {
133  Arena.Reset();
134  llvm::yaml::Input Yin(Value, &Arena);
135  auto Sym = clang::clangd::SymbolFromYAML(Yin, Arena);
137  Key >> ID;
138  if (const auto *Existing = UniqueSymbols.find(ID))
139  UniqueSymbols.insert(mergeSymbol(*Existing, Sym, &Scratch));
140  else
141  UniqueSymbols.insert(Sym);
142  });
143  return std::move(UniqueSymbols).build();
144 }
145 
146 } // namespace
147 } // namespace clangd
148 } // namespace clang
149 
150 int main(int argc, const char **argv) {
151  llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
152 
153  const char *Overview = R"(
154  This is an **experimental** tool to generate YAML-format project-wide symbols
155  for clangd (global code completion). It would be changed and deprecated
156  eventually. Don't use it in production code!
157 
158  Example usage for building index for the whole project using CMake compile
159  commands:
160 
161  $ global-symbol-builder --executor=all-TUs compile_commands.json > index.yaml
162 
163  Example usage for file sequence index without flags:
164 
165  $ global-symbol-builder File1.cpp File2.cpp ... FileN.cpp > index.yaml
166 
167  Note: only symbols from header files will be collected.
168  )";
169 
170  auto Executor = clang::tooling::createExecutorFromCommandLineArgs(
171  argc, argv, cl::GeneralCategory, Overview);
172 
173  if (!Executor) {
174  llvm::errs() << llvm::toString(Executor.takeError()) << "\n";
175  return 1;
176  }
177 
178  if (!clang::clangd::AssumedHeaderDir.empty() &&
179  !llvm::sys::path::is_absolute(clang::clangd::AssumedHeaderDir)) {
180  llvm::errs() << "--assume-header-dir must be an absolute path.\n";
181  return 1;
182  }
183 
184  // Map phase: emit symbols found in each translation unit.
185  auto Err = Executor->get()->execute(
186  llvm::make_unique<clang::clangd::SymbolIndexActionFactory>(
187  Executor->get()->getExecutionContext()));
188  if (Err) {
189  llvm::errs() << llvm::toString(std::move(Err)) << "\n";
190  }
191 
192  // Reduce phase: combine symbols using the ID as a key.
193  auto UniqueSymbols =
194  clang::clangd::mergeSymbols(Executor->get()->getToolResults());
195 
196  // Output phase: emit YAML for result symbols.
197  SymbolsToYAML(UniqueSymbols, llvm::outs());
198  return 0;
199 }
std::unique_ptr< CommentHandler > collectIWYUHeaderMaps(CanonicalIncludes *Includes)
Returns a CommentHandler that parses pragma comment on include files to determine when we should incl...
Some operations such as code completion produce a set of candidates.
static llvm::StringRef toString(SpecialMemberFunctionsCheck::SpecialMemberFunctionKind K)
std::vector< CodeCompletionResult > Results
int main(int argc, const char **argv)
std::unique_ptr< Iterator > create(PostingListRef Documents)
Returns a document iterator over given PostingList.
Definition: Iterator.cpp:228
void SymbolsToYAML(const SymbolSlab &Symbols, llvm::raw_ostream &OS)
Definition: SymbolYAML.cpp:192
std::string SymbolToYAML(Symbol Sym)
Definition: SymbolYAML.cpp:198
Symbol mergeSymbol(const Symbol &L, const Symbol &R, Symbol::Details *Scratch)
Definition: Merge.cpp:83
Symbol SymbolFromYAML(llvm::yaml::Input &Input, llvm::BumpPtrAllocator &Arena)
Definition: SymbolYAML.cpp:184
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
tooling::ExecutionContext * Ctx
void addSystemHeadersMapping(CanonicalIncludes *Includes)
Adds mapping for system headers and some special symbols (e.g.