48 #include "clang/Frontend/CompilerInvocation.h" 49 #include "clang/Frontend/PCHContainerOperations.h" 50 #include "llvm/ADT/ScopeExit.h" 51 #include "llvm/Support/Errc.h" 52 #include "llvm/Support/Path.h" 60 using std::chrono::steady_clock;
72 using Key =
const ASTWorker *;
74 ASTCache(
unsigned MaxRetainedASTs) : MaxRetainedASTs(MaxRetainedASTs) {}
79 std::lock_guard<std::mutex> Lock(Mut);
80 auto It = findByKey(K);
81 if (It == LRU.end() || !It->second)
83 return It->second->getUsedBytes();
88 void put(
Key K, std::unique_ptr<ParsedAST> V) {
89 std::unique_lock<std::mutex> Lock(Mut);
90 assert(findByKey(K) == LRU.end());
92 LRU.insert(LRU.begin(), {K, std::move(V)});
93 if (LRU.size() <= MaxRetainedASTs)
96 std::unique_ptr<ParsedAST> ForCleanup = std::move(LRU.back().second);
106 llvm::Optional<std::unique_ptr<ParsedAST>>
take(
Key K) {
107 std::unique_lock<std::mutex> Lock(Mut);
108 auto Existing = findByKey(K);
109 if (Existing == LRU.end())
111 std::unique_ptr<ParsedAST> V = std::move(Existing->second);
116 return llvm::Optional<std::unique_ptr<ParsedAST>>(std::move(V));
120 using KVPair = std::pair<Key, std::unique_ptr<ParsedAST>>;
122 std::vector<KVPair>::iterator findByKey(
Key K) {
123 return std::find_if(LRU.begin(), LRU.end(),
124 [K](
const KVPair &P) {
return P.first == K; });
128 unsigned MaxRetainedASTs;
131 std::vector<KVPair> LRU;
135 class ASTWorkerHandle;
148 friend class ASTWorkerHandle;
151 steady_clock::duration UpdateDebounce,
152 std::shared_ptr<PCHContainerOperations>
PCHs,
153 bool StorePreamblesInMemory,
165 steady_clock::duration UpdateDebounce,
166 std::shared_ptr<PCHContainerOperations> PCHs,
167 bool StorePreamblesInMemory,
172 llvm::unique_function<
void(std::vector<Diag>)> OnUpdated);
175 llvm::unique_function<
void(llvm::Expected<InputsAndAST>)>
Action);
178 std::shared_ptr<const PreambleData> getPossiblyStalePreamble()
const;
183 void waitForFirstPreamble()
const;
186 bool isASTCached()
const;
196 void startTask(llvm::StringRef Name, llvm::unique_function<
void()> Task,
204 bool shouldSkipHeadLocked()
const;
215 TUScheduler::ASTCache &IdleASTs;
218 const steady_clock::duration UpdateDebounce;
222 const bool StorePreambleInMemory;
226 const std::shared_ptr<PCHContainerOperations>
PCHs;
230 ParseInputs FileInputs;
233 bool DiagsWereReported =
false;
236 mutable std::mutex Mutex;
237 std::shared_ptr<const PreambleData> LastBuiltPreamble;
239 Notification PreambleWasBuilt;
242 std::deque<Request> Requests;
243 mutable std::condition_variable RequestsCV;
250 class ASTWorkerHandle {
251 friend class ASTWorker;
252 ASTWorkerHandle(std::shared_ptr<ASTWorker> Worker)
253 : Worker(std::move(Worker)) {
254 assert(this->Worker);
258 ASTWorkerHandle(
const ASTWorkerHandle &) =
delete;
259 ASTWorkerHandle &operator=(
const ASTWorkerHandle &) =
delete;
260 ASTWorkerHandle(ASTWorkerHandle &&) =
default;
261 ASTWorkerHandle &operator=(ASTWorkerHandle &&) =
default;
268 ASTWorker &operator*() {
269 assert(Worker &&
"Handle was moved from");
273 ASTWorker *operator->() {
274 assert(Worker &&
"Handle was moved from");
282 std::shared_ptr<const ASTWorker> lock() {
return Worker; }
285 std::shared_ptr<ASTWorker> Worker;
289 TUScheduler::ASTCache &IdleASTs,
290 AsyncTaskRunner *Tasks, Semaphore &Barrier,
291 steady_clock::duration UpdateDebounce,
292 std::shared_ptr<PCHContainerOperations>
PCHs,
293 bool StorePreamblesInMemory,
295 std::shared_ptr<ASTWorker> Worker(
new ASTWorker(
296 FileName, IdleASTs, Barrier, !Tasks, UpdateDebounce,
297 std::move(PCHs), StorePreamblesInMemory, std::move(PreambleCallback)));
299 Tasks->runAsync(
"worker:" + llvm::sys::path::filename(FileName),
300 [Worker]() { Worker->run(); });
302 return ASTWorkerHandle(std::move(Worker));
305 ASTWorker::ASTWorker(
PathRef FileName, TUScheduler::ASTCache &LRUCache,
306 Semaphore &Barrier,
bool RunSync,
307 steady_clock::duration UpdateDebounce,
308 std::shared_ptr<PCHContainerOperations> PCHs,
309 bool StorePreamblesInMemory,
311 : IdleASTs(LRUCache), RunSync(RunSync), UpdateDebounce(UpdateDebounce),
312 FileName(FileName), StorePreambleInMemory(StorePreamblesInMemory),
313 PreambleCallback(std::move(PreambleCallback)),
PCHs(std::move(PCHs)),
314 Barrier(Barrier), Done(
false) {}
316 ASTWorker::~ASTWorker() {
320 std::lock_guard<std::mutex> Lock(Mutex);
321 assert(Done &&
"handle was not destroyed");
322 assert(Requests.empty() &&
"unprocessed requests when destroying ASTWorker");
326 void ASTWorker::update(
328 llvm::unique_function<
void(std::vector<Diag>)> OnUpdated) {
329 auto Task = [=](decltype(OnUpdated) OnUpdated)
mutable {
331 bool InputsAreTheSame =
332 std::tie(FileInputs.CompileCommand, FileInputs.Contents) ==
333 std::tie(Inputs.CompileCommand, Inputs.Contents);
335 tooling::CompileCommand OldCommand = std::move(FileInputs.CompileCommand);
336 bool PrevDiagsWereReported = DiagsWereReported;
338 DiagsWereReported =
false;
340 log(
"Updating file {0} with command [{1}] {2}", FileName,
341 Inputs.CompileCommand.Directory,
342 llvm::join(Inputs.CompileCommand.CommandLine,
" "));
344 std::unique_ptr<CompilerInvocation> Invocation =
347 elog(
"Could not build CompilerInvocation for file {0}", FileName);
352 PreambleWasBuilt.notify();
356 std::shared_ptr<const PreambleData> OldPreamble =
357 getPossiblyStalePreamble();
358 std::shared_ptr<const PreambleData> NewPreamble =
359 buildPreamble(FileName, *Invocation, OldPreamble, OldCommand, Inputs,
360 PCHs, StorePreambleInMemory, PreambleCallback);
362 bool CanReuseAST = InputsAreTheSame && (OldPreamble == NewPreamble);
364 std::lock_guard<std::mutex> Lock(Mutex);
366 LastBuiltPreamble = NewPreamble;
372 PreambleWasBuilt.notify();
379 if (PrevDiagsWereReported) {
380 DiagsWereReported =
true;
388 log(
"Skipping rebuild of the AST for {0}, inputs are the same.",
399 llvm::Optional<std::unique_ptr<ParsedAST>>
AST = IdleASTs.take(
this);
401 llvm::Optional<ParsedAST> NewAST =
402 buildAST(FileName, std::move(Invocation), Inputs, NewPreamble, PCHs);
403 AST = NewAST ? llvm::make_unique<ParsedAST>(std::move(*NewAST)) :
nullptr;
410 OnUpdated((*AST)->getDiagnostics());
411 DiagsWereReported =
true;
414 IdleASTs.put(
this, std::move(*AST));
417 startTask(
"Update",
Bind(Task, std::move(OnUpdated)), WantDiags);
420 void ASTWorker::runWithAST(
421 llvm::StringRef
Name,
422 llvm::unique_function<
void(llvm::Expected<InputsAndAST>)>
Action) {
424 llvm::Optional<std::unique_ptr<ParsedAST>>
AST = IdleASTs.take(
this);
426 std::unique_ptr<CompilerInvocation> Invocation =
429 llvm::Optional<ParsedAST> NewAST =
432 llvm::make_unique<CompilerInvocation>(*Invocation),
433 FileInputs, getPossiblyStalePreamble(), PCHs)
435 AST = NewAST ? llvm::make_unique<ParsedAST>(std::move(*NewAST)) :
nullptr;
438 auto _ = llvm::make_scope_exit(
439 [&AST,
this]() { IdleASTs.put(
this, std::move(*AST)); });
442 return Action(llvm::make_error<llvm::StringError>(
443 "invalid AST", llvm::errc::invalid_argument));
444 Action(InputsAndAST{FileInputs, **AST});
446 startTask(Name,
Bind(Task, std::move(
Action)),
450 std::shared_ptr<const PreambleData>
451 ASTWorker::getPossiblyStalePreamble()
const {
452 std::lock_guard<std::mutex> Lock(Mutex);
453 return LastBuiltPreamble;
456 void ASTWorker::waitForFirstPreamble()
const {
457 PreambleWasBuilt.wait();
460 std::size_t ASTWorker::getUsedBytes()
const {
464 std::size_t Result = IdleASTs.getUsedBytes(
this);
465 if (
auto Preamble = getPossiblyStalePreamble())
466 Result +=
Preamble->Preamble.getSize();
470 bool ASTWorker::isASTCached()
const {
return IdleASTs.getUsedBytes(
this) != 0; }
472 void ASTWorker::stop() {
474 std::lock_guard<std::mutex> Lock(Mutex);
475 assert(!Done &&
"stop() called twice");
478 RequestsCV.notify_all();
481 void ASTWorker::startTask(llvm::StringRef Name,
482 llvm::unique_function<
void()> Task,
485 assert(!Done &&
"running a task after stop()");
486 trace::Span Tracer(Name +
":" + llvm::sys::path::filename(FileName));
492 std::lock_guard<std::mutex> Lock(Mutex);
493 assert(!Done &&
"running a task after stop()");
494 Requests.push_back({std::move(Task),
Name, steady_clock::now(),
497 RequestsCV.notify_all();
500 void ASTWorker::run() {
504 std::unique_lock<std::mutex> Lock(Mutex);
505 for (
auto Wait = scheduleLocked(); !Wait.expired();
506 Wait = scheduleLocked()) {
508 if (Requests.empty())
515 Optional<WithContext>
Ctx;
516 Optional<trace::Span> Tracer;
517 if (!Requests.empty()) {
518 Ctx.emplace(Requests.front().Ctx.clone());
519 Tracer.emplace(
"Debounce");
520 SPAN_ATTACH(*Tracer,
"next_request", Requests.front().Name);
523 std::chrono::duration_cast<std::chrono::milliseconds>(
524 Wait.time() - steady_clock::now())
528 wait(Lock, RequestsCV, Wait);
530 Req = std::move(Requests.front());
535 std::lock_guard<Semaphore> BarrierLock(Barrier);
536 WithContext Guard(std::move(Req.Ctx));
537 trace::Span Tracer(Req.Name);
542 std::lock_guard<std::mutex> Lock(Mutex);
543 Requests.pop_front();
545 RequestsCV.notify_all();
549 Deadline ASTWorker::scheduleLocked() {
550 if (Requests.empty())
552 while (shouldSkipHeadLocked())
553 Requests.pop_front();
554 assert(!Requests.empty() &&
"skipped the whole queue");
559 for (
const auto &R : Requests)
563 Deadline D(Requests.front().AddTime + UpdateDebounce);
568 bool ASTWorker::shouldSkipHeadLocked()
const {
569 assert(!Requests.empty());
570 auto Next = Requests.begin();
571 auto UpdateType = Next->UpdateType;
577 if (Next == Requests.end() || !Next->UpdateType)
580 switch (*UpdateType) {
587 for (; Next != Requests.end(); ++Next)
593 llvm_unreachable(
"Unknown WantDiagnostics");
596 bool ASTWorker::blockUntilIdle(Deadline Timeout)
const {
597 std::unique_lock<std::mutex> Lock(Mutex);
598 return wait(Lock, RequestsCV, Timeout, [&] {
return Requests.empty(); });
604 unsigned HardwareConcurrency = std::thread::hardware_concurrency();
608 if (HardwareConcurrency == 0)
610 return HardwareConcurrency;
621 bool StorePreamblesInMemory,
623 std::chrono::steady_clock::duration UpdateDebounce,
625 : StorePreamblesInMemory(StorePreamblesInMemory),
626 PCHOps(std::make_shared<PCHContainerOperations>()),
627 PreambleCallback(std::move(PreambleCallback)), Barrier(AsyncThreadsCount),
628 IdleASTs(
llvm::make_unique<
ASTCache>(RetentionPolicy.MaxRetainedASTs)),
629 UpdateDebounce(UpdateDebounce) {
630 if (0 < AsyncThreadsCount) {
631 PreambleTasks.emplace();
632 WorkerThreads.emplace();
642 PreambleTasks->wait();
644 WorkerThreads->wait();
648 for (
auto &
File : Files)
649 if (!
File.getValue()->Worker->blockUntilIdle(D))
652 if (!PreambleTasks->wait(D))
659 llvm::unique_function<
void(std::vector<Diag>)> OnUpdated) {
660 std::unique_ptr<FileData> &FD = Files[
File];
664 File, *IdleASTs, WorkerThreads ? WorkerThreads.getPointer() :
nullptr,
665 Barrier, UpdateDebounce, PCHOps, StorePreamblesInMemory,
667 FD = std::unique_ptr<FileData>(
new FileData{
673 FD->Worker->update(std::move(Inputs), WantDiags, std::move(OnUpdated));
677 bool Removed = Files.erase(File);
679 elog(
"Trying to remove file from TUScheduler that is not tracked: {0}",
685 llvm::unique_function<
void(llvm::Expected<InputsAndAST>)>
Action) {
686 auto It = Files.find(File);
687 if (It == Files.end()) {
688 Action(llvm::make_error<llvm::StringError>(
689 "trying to get AST for non-added document",
690 llvm::errc::invalid_argument));
694 It->second->Worker->runWithAST(Name, std::move(
Action));
698 llvm::StringRef Name,
PathRef File,
699 llvm::unique_function<
void(llvm::Expected<InputsAndPreamble>)>
Action) {
700 auto It = Files.find(File);
701 if (It == Files.end()) {
702 Action(llvm::make_error<llvm::StringError>(
703 "trying to get preamble for non-added document",
704 llvm::errc::invalid_argument));
708 if (!PreambleTasks) {
711 std::shared_ptr<const PreambleData>
Preamble =
712 It->second->Worker->getPossiblyStalePreamble();
718 std::shared_ptr<const ASTWorker> Worker = It->second->Worker.lock();
719 auto Task = [Worker,
this](std::string
Name, std::string
File,
726 Worker->waitForFirstPreamble();
728 std::lock_guard<Semaphore> BarrierLock(Barrier);
732 std::shared_ptr<const PreambleData>
Preamble =
733 Worker->getPossiblyStalePreamble();
737 PreambleTasks->runAsync(
"task:" + llvm::sys::path::filename(File),
738 Bind(Task, std::string(Name), std::string(File),
739 It->second->Contents, It->second->Command,
743 std::vector<std::pair<Path, std::size_t>>
745 std::vector<std::pair<Path, std::size_t>> Result;
746 Result.reserve(Files.size());
747 for (
auto &&PathAndFile : Files)
749 {PathAndFile.first(), PathAndFile.second->Worker->getUsedBytes()});
754 std::vector<Path> Result;
755 for (
auto &&PathAndFile : Files) {
756 if (!PathAndFile.second->Worker->isASTCached())
758 Result.push_back(PathAndFile.first());
const tooling::CompileCommand & Command
WantDiagnostics
Determines whether diagnostics should be generated for a file snapshot.
Some operations such as code completion produce a set of candidates.
Diagnostics must be generated for this snapshot.
std::function< void(PathRef Path, ASTContext &, std::shared_ptr< clang::Preprocessor >)> PreambleParsedCallback
bool blockUntilIdle(Deadline D) const
Wait until there are no scheduled or running tasks.
llvm::Optional< ParsedAST > buildAST(PathRef FileName, std::unique_ptr< CompilerInvocation > Invocation, const ParseInputs &Inputs, std::shared_ptr< const PreambleData > Preamble, std::shared_ptr< PCHContainerOperations > PCHs)
Build an AST from provided user inputs.
void remove(PathRef File)
Remove File from the list of tracked files and schedule removal of its resources. ...
llvm::StringRef PathRef
A typedef to represent a ref to file path.
Documents should not be synced at all.
Limits the number of threads that can acquire the lock at the same time.
void elog(const char *Fmt, Ts &&... Vals)
Configuration of the AST retention policy.
Context clone() const
Clone this context object.
std::unique_ptr< Iterator > create(PostingListRef Documents)
Returns a document iterator over given PostingList.
ForwardBinder< Func, Args... > Bind(Func F, Args &&... As)
Creates an object that stores a callable (F) and first arguments to the callable (As) and allows to c...
llvm::Optional< WantDiagnostics > UpdateType
static Deadline infinity()
llvm::unique_function< void()> Action
void log(const char *Fmt, Ts &&... Vals)
void put(Key K, std::unique_ptr< ParsedAST > V)
Store the value in the pool, possibly removing the last used AST.
std::string Path
A typedef to represent a file path.
static const Context & current()
Returns the context for the current thread, creating it if needed.
std::unique_ptr< CompilerInvocation > buildCompilerInvocation(const ParseInputs &Inputs)
Builds compiler invocation that could be used to build AST or preamble.
std::vector< std::pair< Path, std::size_t > > getUsedBytesPerFile() const
Returns estimated memory usage for each of the currently open files.
std::vector< Path > getFilesWithCachedAST() const
Returns a list of files with ASTs currently stored in memory.
std::shared_ptr< const PreambleData > buildPreamble(PathRef FileName, CompilerInvocation &CI, std::shared_ptr< const PreambleData > OldPreamble, const tooling::CompileCommand &OldCompileCommand, const ParseInputs &Inputs, std::shared_ptr< PCHContainerOperations > PCHs, bool StoreInMemory, PreambleParsedCallback PreambleCallback)
Rebuild the preamble for the new inputs unless the old one can be reused.
void wait(std::unique_lock< std::mutex > &Lock, std::condition_variable &CV, Deadline D)
Wait once on CV for the specified duration.
Runs tasks on separate (detached) threads and wait for all tasks to finish.
PrecompiledPreamble const * Preamble
A context is an immutable container for per-request data that must be propagated through layers that ...
void update(PathRef File, ParseInputs Inputs, WantDiagnostics WD, llvm::unique_function< void(std::vector< Diag >)> OnUpdated)
Schedule an update for File.
unsigned getDefaultAsyncThreadsCount()
Returns a number of a default async threads to use for TUScheduler.
WithContext replaces Context::current() with a provided scope.
std::string Contents
Latest inputs, passed to TUScheduler::update().
ASTCache(unsigned MaxRetainedASTs)
void runWithPreamble(llvm::StringRef Name, PathRef File, Callback< InputsAndPreamble > Action)
Schedule an async read of the Preamble.
===– Representation.cpp - ClangDoc Representation --------—*- C++ -*-===//
llvm::Optional< std::unique_ptr< ParsedAST > > take(Key K)
Returns the cached value for K, or llvm::None if the value is not in the cache anymore.
std::shared_ptr< PCHContainerOperations > PCHs
steady_clock::time_point AddTime
tooling::CompileCommand Command
void runWithAST(llvm::StringRef Name, PathRef File, Callback< InputsAndAST > Action)
Schedule an async read of the AST.
A point in time we can wait for.
static std::string join(ArrayRef< SpecialMemberFunctionsCheck::SpecialMemberFunctionKind > SMFS, llvm::StringRef AndOr)
An LRU cache of idle ASTs.
Records an event whose duration is the lifetime of the Span object.
TUScheduler(unsigned AsyncThreadsCount, bool StorePreamblesInMemory, PreambleParsedCallback PreambleCallback, std::chrono::steady_clock::duration UpdateDebounce, ASTRetentionPolicy RetentionPolicy)
#define SPAN_ATTACH(S, Name, Expr)
Attach a key-value pair to a Span event.
Diagnostics must not be generated for this snapshot.
std::size_t getUsedBytes(Key K)
Returns result of getUsedBytes() for the AST cached by K.