Cache calculated file state in LSP (#4897)

Add caching of parsed documents, and testing of the textDocument handlers. This is based on #4896, which splits out some of the boilerplate to calls. Note, this caches the entire parse state because we'll want to try to emit diagnostics when we see the update, without waiting. It may be helpful to do that asynchronously, but we don't want to wait for another call (such as documentSymbol). Really, we'll probably want to also add check for diagnostics, at least.
carbon-language · Feb 7, 2025 · 9e466b9 · 9e466b9
1 parent 55714dd
commit 9e466b9
Show file tree

Hide file tree

Showing 28 changed files with 498 additions and 51 deletions.
diff --git a/toolchain/diagnostics/diagnostic_kind.def b/toolchain/diagnostics/diagnostic_kind.def
@@ -28,11 +28,6 @@ CARBON_DIAGNOSTIC_KIND(CompilePreludeManifestError)
 CARBON_DIAGNOSTIC_KIND(CompileInputNotRegularFile)
 CARBON_DIAGNOSTIC_KIND(CompileOutputFileOpenError)
 CARBON_DIAGNOSTIC_KIND(FormatMultipleFilesToOneOutput)
-CARBON_DIAGNOSTIC_KIND(LanguageServerMissingInputStream)
-CARBON_DIAGNOSTIC_KIND(LanguageServerNotificationParseError)
-CARBON_DIAGNOSTIC_KIND(LanguageServerTransportError)
-CARBON_DIAGNOSTIC_KIND(LanguageServerUnexpectedReply)
-CARBON_DIAGNOSTIC_KIND(LanguageServerUnsupportedNotification)
 
 // ============================================================================
 // SourceBuffer diagnostics
@@ -441,6 +436,23 @@ CARBON_DIAGNOSTIC_KIND(AssociatedConstantWithDifferentValues)
 CARBON_DIAGNOSTIC_KIND(ImplsOnNonFacetType)
 CARBON_DIAGNOSTIC_KIND(WhereOnNonFacetType)
 
+// ============================================================================
+// Language server diagnostics
+// ============================================================================
+
+CARBON_DIAGNOSTIC_KIND(LanguageServerFileUnknown)
+CARBON_DIAGNOSTIC_KIND(LanguageServerFileUnsupported)
+CARBON_DIAGNOSTIC_KIND(LanguageServerMissingInputStream)
+CARBON_DIAGNOSTIC_KIND(LanguageServerNotificationParseError)
+CARBON_DIAGNOSTIC_KIND(LanguageServerTransportError)
+CARBON_DIAGNOSTIC_KIND(LanguageServerUnexpectedReply)
+CARBON_DIAGNOSTIC_KIND(LanguageServerUnsupportedNotification)
+
+// Document handling.
+CARBON_DIAGNOSTIC_KIND(LanguageServerOpenDuplicateFile)
+CARBON_DIAGNOSTIC_KIND(LanguageServerUnsupportedChanges)
+CARBON_DIAGNOSTIC_KIND(LanguageServerCloseUnknownFile)
+
 // ============================================================================
 // Other diagnostics
 // ============================================================================

diff --git a/toolchain/language_server/BUILD b/toolchain/language_server/BUILD
@@ -28,10 +28,20 @@ cc_library(
 
 cc_library(
     name = "context",
+    srcs = ["context.cpp"],
     hdrs = ["context.h"],
     deps = [
         "//common:map",
+        "//toolchain/base:shared_value_stores",
         "//toolchain/diagnostics:diagnostic_emitter",
+        "//toolchain/diagnostics:file_diagnostics",
+        "//toolchain/diagnostics:null_diagnostics",
+        "//toolchain/lex",
+        "//toolchain/lex:tokenized_buffer",
+        "//toolchain/parse",
+        "//toolchain/parse:tree",
+        "//toolchain/sem_ir:file",
+        "//toolchain/source:source_buffer",
     ],
 )
 

diff --git a/toolchain/language_server/context.cpp b/toolchain/language_server/context.cpp
@@ -0,0 +1,68 @@
+// Part of the Carbon Language project, under the Apache License v2.0 with LLVM
+// Exceptions. See /LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "toolchain/language_server/context.h"
+
+#include <memory>
+
+#include "toolchain/base/shared_value_stores.h"
+#include "toolchain/diagnostics/null_diagnostics.h"
+#include "toolchain/lex/lex.h"
+#include "toolchain/lex/tokenized_buffer.h"
+#include "toolchain/parse/parse.h"
+#include "toolchain/parse/tree_and_subtrees.h"
+
+namespace Carbon::LanguageServer {
+
+auto Context::File::SetText(Context& context, llvm::StringRef text) -> void {
+  // Clear state dependent on the source text.
+  tree_and_subtrees_.reset();
+  tree_.reset();
+  tokens_.reset();
+  value_stores_.reset();
+  source_.reset();
+
+  // TODO: Make the processing asynchronous, to better handle rapid text
+  // updates.
+  CARBON_CHECK(!source_ && !value_stores_ && !tokens_ && !tree_,
+               "We currently cache everything together");
+  // TODO: Diagnostics should be passed to the LSP instead of dropped.
+  auto& null_consumer = NullDiagnosticConsumer();
+  std::optional source =
+      SourceBuffer::MakeFromStringCopy(filename_, text, null_consumer);
+  if (!source) {
+    // Failing here should be rare, but provide stub data for recovery so that
+    // we can have a simple API.
+    source = SourceBuffer::MakeFromStringCopy(filename_, "", null_consumer);
+    CARBON_CHECK(source, "Making an empty buffer should always succeed");
+  }
+  source_ = std::make_unique<SourceBuffer>(std::move(*source));
+  value_stores_ = std::make_unique<SharedValueStores>();
+  tokens_ = std::make_unique<Lex::TokenizedBuffer>(
+      Lex::Lex(*value_stores_, *source_, null_consumer));
+  tree_ = std::make_unique<Parse::Tree>(
+      Parse::Parse(*tokens_, null_consumer, context.vlog_stream()));
+  tree_and_subtrees_ =
+      std::make_unique<Parse::TreeAndSubtrees>(*tokens_, *tree_);
+}
+
+auto Context::LookupFile(llvm::StringRef filename) -> File* {
+  if (!filename.ends_with(".carbon")) {
+    CARBON_DIAGNOSTIC(LanguageServerFileUnsupported, Warning,
+                      "non-Carbon file requested");
+    file_emitter_.Emit(filename, LanguageServerFileUnsupported);
+    return nullptr;
+  }
+
+  if (auto lookup_result = files().Lookup(filename)) {
+    return &lookup_result.value();
+  } else {
+    CARBON_DIAGNOSTIC(LanguageServerFileUnknown, Warning,
+                      "unknown file requested");
+    file_emitter_.Emit(filename, LanguageServerFileUnknown);
+    return nullptr;
+  }
+}
+
+}  // namespace Carbon::LanguageServer
diff --git a/toolchain/language_server/context.h b/toolchain/language_server/context.h
@@ -5,29 +5,72 @@
 #ifndef CARBON_TOOLCHAIN_LANGUAGE_SERVER_CONTEXT_H_
 #define CARBON_TOOLCHAIN_LANGUAGE_SERVER_CONTEXT_H_
 
+#include <memory>
 #include <string>
 
 #include "common/map.h"
+#include "toolchain/base/shared_value_stores.h"
 #include "toolchain/diagnostics/diagnostic_consumer.h"
 #include "toolchain/diagnostics/diagnostic_emitter.h"
+#include "toolchain/diagnostics/file_diagnostics.h"
+#include "toolchain/lex/tokenized_buffer.h"
+#include "toolchain/parse/tree_and_subtrees.h"
+#include "toolchain/sem_ir/file.h"
+#include "toolchain/source/source_buffer.h"
 
 namespace Carbon::LanguageServer {
 
 // Context for LSP call handling.
 class Context {
  public:
-  // `consumer` is required.
-  explicit Context(DiagnosticConsumer* consumer) : no_loc_emitter_(consumer) {}
+  // Cached information for an open file.
+  class File {
+   public:
+    explicit File(std::string filename) : filename_(std::move(filename)) {}
 
+    // Changes the file's text, updating dependent state.
+    auto SetText(Context& context, llvm::StringRef text) -> void;
+
+    auto tree_and_subtrees() const -> const Parse::TreeAndSubtrees& {
+      return *tree_and_subtrees_;
+    }
+
+   private:
+    // The filename, stable across instances.
+    std::string filename_;
+
+    // Current file content, and derived values.
+    std::unique_ptr<SourceBuffer> source_;
+    std::unique_ptr<SharedValueStores> value_stores_;
+    std::unique_ptr<Lex::TokenizedBuffer> tokens_;
+    std::unique_ptr<Parse::Tree> tree_;
+    std::unique_ptr<Parse::TreeAndSubtrees> tree_and_subtrees_;
+  };
+
+  // `consumer` and `emitter` are required. `vlog_stream` is optional.
+  explicit Context(llvm::raw_ostream* vlog_stream, DiagnosticConsumer* consumer)
+      : vlog_stream_(vlog_stream),
+        file_emitter_(consumer),
+        no_loc_emitter_(consumer) {}
+
+  // Returns a reference to the file if it's known, or diagnoses and returns
+  // null.
+  auto LookupFile(llvm::StringRef filename) -> File*;
+
+  auto file_emitter() -> FileDiagnosticEmitter& { return file_emitter_; }
   auto no_loc_emitter() -> NoLocDiagnosticEmitter& { return no_loc_emitter_; }
+  auto vlog_stream() -> llvm::raw_ostream* { return vlog_stream_; }
 
-  auto files() -> Map<std::string, std::string>& { return files_; }
+  auto files() -> Map<std::string, File>& { return files_; }
 
  private:
+  // Diagnostic and output streams.
+  llvm::raw_ostream* vlog_stream_;
+  FileDiagnosticEmitter file_emitter_;
   NoLocDiagnosticEmitter no_loc_emitter_;
 
   // Content of files managed by the language client.
-  Map<std::string, std::string> files_;
+  Map<std::string, File> files_;
 };
 
 }  // namespace Carbon::LanguageServer

diff --git a/toolchain/language_server/handle.h b/toolchain/language_server/handle.h
@@ -15,6 +15,11 @@ auto HandleDidChangeTextDocument(
     Context& context, const clang::clangd::DidChangeTextDocumentParams& params)
     -> void;
 
+// Closes a document.
+auto HandleDidCloseTextDocument(
+    Context& context, const clang::clangd::DidCloseTextDocumentParams& params)
+    -> void;
+
 // Updates the content of already-open documents.
 auto HandleDidOpenTextDocument(
     Context& context, const clang::clangd::DidOpenTextDocumentParams& params)

diff --git a/toolchain/language_server/handle_document_symbol.cpp b/toolchain/language_server/handle_document_symbol.cpp
@@ -15,18 +15,17 @@ namespace Carbon::LanguageServer {
 
 // Returns the text of first child of kind IdentifierNameBeforeParams or
 // IdentifierNameNotBeforeParams.
-static auto GetIdentifierName(const SharedValueStores& value_stores,
-                              const Lex::TokenizedBuffer& tokens,
-                              const Parse::TreeAndSubtrees& tree_and_subtrees,
+static auto GetIdentifierName(const Parse::TreeAndSubtrees& tree_and_subtrees,
                               Parse::NodeId node)
     -> std::optional<llvm::StringRef> {
+  const auto& tokens = tree_and_subtrees.tree().tokens();
   for (auto child : tree_and_subtrees.children(node)) {
     switch (tree_and_subtrees.tree().node_kind(child)) {
       case Parse::NodeKind::IdentifierNameBeforeParams:
       case Parse::NodeKind::IdentifierNameNotBeforeParams: {
         auto token = tree_and_subtrees.tree().node_token(child);
         if (tokens.GetKind(token) == Lex::TokenKind::Identifier) {
-          return value_stores.identifiers().Get(tokens.GetIdentifier(token));
+          return tokens.GetTokenText(token);
         }
         break;
       }
@@ -42,22 +41,19 @@ auto HandleDocumentSymbol(
     llvm::function_ref<
         void(llvm::Expected<std::vector<clang::clangd::DocumentSymbol>>)>
         on_done) -> void {
-  SharedValueStores value_stores;
-  llvm::vfs::InMemoryFileSystem vfs;
-  auto lookup = context.files().Lookup(params.textDocument.uri.file());
-  CARBON_CHECK(lookup);
-  vfs.addFile(lookup.key(), /*mtime=*/0,
-              llvm::MemoryBuffer::getMemBufferCopy(lookup.value()));
+  auto* file = context.LookupFile(params.textDocument.uri.file());
+  if (!file) {
+    return;
+  }
+
+  const auto& tree_and_subtrees = file->tree_and_subtrees();
+  const auto& tree = tree_and_subtrees.tree();
+  const auto& tokens = tree.tokens();
 
-  auto source =
-      SourceBuffer::MakeFromFile(vfs, lookup.key(), NullDiagnosticConsumer());
-  auto tokens = Lex::Lex(value_stores, *source, NullDiagnosticConsumer());
-  auto tree = Parse::Parse(tokens, NullDiagnosticConsumer(), nullptr);
-  Parse::TreeAndSubtrees tree_and_subtrees(tokens, tree);
   std::vector<clang::clangd::DocumentSymbol> result;
-  for (const auto& node : tree.postorder()) {
+  for (const auto& node_id : tree.postorder()) {
     clang::clangd::SymbolKind symbol_kind;
-    switch (tree.node_kind(node)) {
+    switch (tree.node_kind(node_id)) {
       case Parse::NodeKind::FunctionDecl:
       case Parse::NodeKind::FunctionDefinitionStart:
         symbol_kind = clang::clangd::SymbolKind::Function;
@@ -76,9 +72,8 @@ auto HandleDocumentSymbol(
         continue;
     }
 
-    if (auto name =
-            GetIdentifierName(value_stores, tokens, tree_and_subtrees, node)) {
-      auto token = tree.node_token(node);
+    if (auto name = GetIdentifierName(tree_and_subtrees, node_id)) {
+      auto token = tree.node_token(node_id);
       clang::clangd::Position pos{tokens.GetLineNumber(token) - 1,
                                   tokens.GetColumnNumber(token) - 1};
 

diff --git a/toolchain/language_server/handle_text_document.cpp b/toolchain/language_server/handle_text_document.cpp
@@ -9,17 +9,58 @@ namespace Carbon::LanguageServer {
 auto HandleDidOpenTextDocument(
     Context& context, const clang::clangd::DidOpenTextDocumentParams& params)
     -> void {
-  context.files().Update(params.textDocument.uri.file(),
-                         params.textDocument.text);
+  llvm::StringRef filename = params.textDocument.uri.file();
+  if (!filename.ends_with(".carbon")) {
+    // Ignore non-Carbon files.
+    return;
+  }
+
+  auto insert_result = context.files().Insert(
+      filename, [&] { return Context::File(filename.str()); });
+  insert_result.value().SetText(context, params.textDocument.text);
+  if (!insert_result.is_inserted()) {
+    CARBON_DIAGNOSTIC(LanguageServerOpenDuplicateFile, Warning,
+                      "duplicate open file request; updating content");
+    context.file_emitter().Emit(filename, LanguageServerOpenDuplicateFile);
+  }
 }
 
 auto HandleDidChangeTextDocument(
     Context& context, const clang::clangd::DidChangeTextDocumentParams& params)
     -> void {
+  llvm::StringRef filename = params.textDocument.uri.file();
+  if (!filename.ends_with(".carbon")) {
+    // Ignore non-Carbon files.
+    return;
+  }
+
   // Full text is sent if full sync is specified in capabilities.
-  CARBON_CHECK(params.contentChanges.size() == 1);
-  context.files().Update(params.textDocument.uri.file(),
-                         params.contentChanges[0].text);
+  if (params.contentChanges.size() != 1) {
+    CARBON_DIAGNOSTIC(LanguageServerUnsupportedChanges, Warning,
+                      "received unsupported contentChanges count: {0}", int);
+    context.file_emitter().Emit(filename, LanguageServerUnsupportedChanges,
+                                params.contentChanges.size());
+    return;
+  }
+  if (auto* file = context.LookupFile(filename)) {
+    file->SetText(context, params.contentChanges[0].text);
+  }
+}
+
+auto HandleDidCloseTextDocument(
+    Context& context, const clang::clangd::DidCloseTextDocumentParams& params)
+    -> void {
+  llvm::StringRef filename = params.textDocument.uri.file();
+  if (!filename.ends_with(".carbon")) {
+    // Ignore non-Carbon files.
+    return;
+  }
+
+  if (!context.files().Erase(filename)) {
+    CARBON_DIAGNOSTIC(LanguageServerCloseUnknownFile, Warning,
+                      "tried closing unknown file; ignoring request");
+    context.file_emitter().Emit(filename, LanguageServerCloseUnknownFile);
+  }
 }
 
 }  // namespace Carbon::LanguageServer
diff --git a/toolchain/language_server/incoming_messages.cpp b/toolchain/language_server/incoming_messages.cpp
@@ -76,6 +76,7 @@ IncomingMessages::IncomingMessages(clang::clangd::Transport* transport,
   AddCallHandler("initialize", &HandleInitialize);
   AddNotificationHandler("textDocument/didChange",
                          &HandleDidChangeTextDocument);
+  AddNotificationHandler("textDocument/didClose", &HandleDidCloseTextDocument);
   AddNotificationHandler("textDocument/didOpen", &HandleDidOpenTextDocument);
 }
 

diff --git a/toolchain/language_server/language_server.cpp b/toolchain/language_server/language_server.cpp
@@ -55,7 +55,7 @@ auto Run(FILE* input_stream, llvm::raw_ostream& output_stream,
       clang::clangd::newJSONTransport(input_stream, output_stream,
                                       /*InMirror=*/nullptr,
                                       /*Pretty=*/true));
-  Context context(&consumer);
+  Context context(vlog_stream, &consumer);
   // TODO: Use error_stream in IncomingMessages to report dropped errors.
   IncomingMessages incoming(transport.get(), &context);
   OutgoingMessages outgoing(transport.get());

diff --git a/...hain/language_server/testdata/exit.carbon → ...nguage_server/testdata/basics/exit.carbon b/...hain/language_server/testdata/exit.carbon → ...nguage_server/testdata/basics/exit.carbon
@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/exit.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/exit.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/exit.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/exit.carbon
 
 // --- STDIN
 [[@LSP-NOTIFY:exit]]

diff --git a/...e_server/testdata/fail_empty_stdin.carbon → ...r/testdata/basics/fail_empty_stdin.carbon b/...e_server/testdata/fail_empty_stdin.carbon → ...r/testdata/basics/fail_empty_stdin.carbon
@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/fail_empty_stdin.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/fail_empty_stdin.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/fail_empty_stdin.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/fail_empty_stdin.carbon
 
 // --- STDIN
 // --- AUTOUPDATE-SPLIT

diff --git a/...uage_server/testdata/fail_no_stdin.carbon → ...rver/testdata/basics/fail_no_stdin.carbon b/...uage_server/testdata/fail_no_stdin.carbon → ...rver/testdata/basics/fail_no_stdin.carbon
@@ -4,9 +4,9 @@
 //
 // AUTOUPDATE
 // TIP: To test this file alone, run:
-// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/fail_no_stdin.carbon
+// TIP:   bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/basics/fail_no_stdin.carbon
 // TIP: To dump output, run:
-// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/fail_no_stdin.carbon
+// TIP:   bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/basics/fail_no_stdin.carbon
 
 // --- AUTOUPDATE-SPLIT