Finish migrating off Document to ArkFile (#1267)

Branched from #1266
Progress towards #1212

With this PR everything now goes through Oak's DB.

- Remove Ark's `Document` structure, along with its copy of the
tree-sitter tree, source code, and line index. These are now all
accessed via Oak queries on `File`.

- `WorldState`'s `documents` field becomes `open_files:
HashMap<FilePath, ArkFile>`. (Maybe we should rename to `ArkOpenFile`?)

Author: lionel-

crates/ark/src/lsp.rs

@@ -12,11 +12,11 @@ pub mod code_action;
 pub mod comm;
 pub mod completions;
 mod config;
+mod content_changes;
 pub(crate) mod db;
 mod declarations;
 pub mod diagnostics;
 pub mod diagnostics_syntax;
-pub mod document;
 pub mod document_context;
 pub mod events;
 pub mod folding_range;

crates/ark/src/lsp/ark_file.rs

@@ -18,15 +18,29 @@ use crate::lsp::db::FileArkExt;
 
 /// Editor-managed buffer state, paired with its `oak_db::File`.
 ///
-/// `ArkFile` and `OakDatabase` are sibling fields on `WorldState`, so an
-/// `ArkFile` cannot hold a reference to the database. That's why the methods
-/// below take `db` as an argument instead of storing a reference, which is the
-/// Salsa convention anyway.
-#[derive(Debug)]
+/// This is a temporary structure during the transition to pure Oak handlers.
+///
+/// The methods take `db` as a parameter rather than holding it. `ArkFile` lives
+/// in `WorldState`, and the db is a sibling field there, so a stored borrow of
+/// it would be self-referential, which safe Rust forbids. Passing `db` per call
+/// is the salsa idiom anyway (`file.parse(db)`).
+#[derive(Clone, Debug)]
 pub(crate) struct ArkFile {
     pub(crate) file: File,
     pub(crate) version: Option<i32>,
     pub(crate) config: DocumentConfig,
+    // The editor's verbatim URL. We store it rather than recompute it from
+    // `file`'s path so the bytes the frontend sent round-trip exactly. It lives
+    // on `ArkFile` so it travels with owned values for callers that can't
+    // easily access `WorldState::open_files`: the diagnostics task on a worker
+    // thread (`RefreshDiagnosticsTask`) and `code_action/roxygen.rs`, which
+    // builds a `WorkspaceEdit` keyed by URL.
+    //
+    // TODO: this is a stopgap that goes away with `ArkFile`. Once handlers are
+    // pure Oak, they return `File`-keyed results (diagnostics, the edit targets
+    // in a `WorkspaceEdit`) and the wire URL gets attached at the transport
+    // boundary from a map of open editor URLs owned by the LSP layer. In that
+    // design the verbatim URL never travels through the analysis layer.
     pub(crate) url: Url,
     pub(crate) encoding: PositionEncoding,
 }
@@ -161,3 +175,60 @@ pub(crate) fn test_ark_file(code: &str) -> (oak_db::OakDatabase, ArkFile) {
     };
     (db, file)
 }
+
+#[cfg(test)]
+mod tests {
+    use tree_sitter::Point;
+
+    use super::*;
+
+    #[test]
+    fn test_tree_sitter_point_from_lsp_position_wide_encoding() {
+        // The emoji is 4 UTF-8 bytes and 2 UTF-16 bytes
+        // `test_ark_file` defaults to UTF-16, the encoding under test here.
+        let (db, ark_file) = test_ark_file("😃a");
+
+        let point = ark_file
+            .tree_sitter_point_from_lsp_position(&db, lsp_types::Position::new(0, 2))
+            .unwrap();
+        assert_eq!(point, Point::new(0, 4));
+
+        let point = ark_file
+            .tree_sitter_point_from_lsp_position(&db, lsp_types::Position::new(0, 3))
+            .unwrap();
+        assert_eq!(point, Point::new(0, 5));
+    }
+
+    #[test]
+    fn test_lsp_position_from_tree_sitter_point_wide_encoding() {
+        let (db, ark_file) = test_ark_file("😃a");
+
+        let position = ark_file
+            .lsp_position_from_tree_sitter_point(&db, Point::new(0, 4))
+            .unwrap();
+        assert_eq!(position, lsp_types::Position::new(0, 2));
+
+        let position = ark_file
+            .lsp_position_from_tree_sitter_point(&db, Point::new(0, 5))
+            .unwrap();
+        assert_eq!(position, lsp_types::Position::new(0, 3));
+    }
+
+    #[test]
+    fn test_utf8_position_roundtrip_multibyte() {
+        // `é` is 2 bytes
+        let (db, mut ark_file) = test_ark_file("é\n");
+        ark_file.encoding = PositionEncoding::Utf8;
+
+        let lsp_position = lsp_types::Position::new(0, 2);
+        let point = ark_file
+            .tree_sitter_point_from_lsp_position(&db, lsp_position)
+            .unwrap();
+        assert_eq!(point, Point::new(0, 2));
+
+        let roundtrip_position = ark_file
+            .lsp_position_from_tree_sitter_point(&db, point)
+            .unwrap();
+        assert_eq!(roundtrip_position, lsp_position);
+    }
+}

crates/ark/src/lsp/content_changes.rs

@@ -0,0 +1,119 @@
+use aether_lsp_utils::proto::from_proto;
+use aether_lsp_utils::proto::PositionEncoding;
+use tower_lsp::lsp_types;
+
+// --- source
+// authors = ["rust-analyzer team"]
+// license = "MIT OR Apache-2.0"
+// origin = "https://github.com/rust-lang/rust-analyzer/blob/master/crates/rust-analyzer/src/lsp/utils.rs"
+// ---
+/// Apply a batch of LSP content changes to `contents`, returning the new text.
+pub(crate) fn apply_content_changes(
+    contents: &str,
+    content_changes: &[lsp_types::TextDocumentContentChangeEvent],
+    encoding: PositionEncoding,
+) -> String {
+    let mut contents = contents.to_string();
+    let mut changes = content_changes.to_vec();
+
+    // If at least one of the changes is a full document change, use the last of them
+    // as the starting point and ignore all previous changes. We then know that all
+    // changes after this (if any!) are incremental changes.
+    //
+    // If we do have a full document change, that implies the `last_start_line`
+    // corresponding to that change is line 0, which will correctly force a rebuild
+    // of the line index before applying any incremental changes.
+    let (changes, mut last_start_line) =
+        match changes.iter().rposition(|change| change.range.is_none()) {
+            Some(idx) => {
+                let incremental = changes.split_off(idx + 1);
+                // Unwrap: `rposition()` confirmed this index contains a full document change
+                let change = changes.pop().unwrap();
+                contents = change.text;
+                (incremental, 0)
+            },
+            None => (changes, u32::MAX),
+        };
+
+    let mut line_index = biome_line_index::LineIndex::new(&contents);
+
+    // Handle all incremental changes after the last full document change. We don't
+    // typically get >1 incremental change as the user types, but we do get them in a
+    // batch after a find-and-replace, or after a format-on-save request.
+    //
+    // Some editors like VS Code send the edits in reverse order (from the bottom of
+    // file -> top of file). We can take advantage of this, because applying an edit
+    // on, say, line 10, doesn't invalidate the `line_index` if we then need to apply
+    // an additional edit on line 5. That said, we may still have edits that cross
+    // lines, so rebuilding the `line_index` is not always unavoidable.
+    for change in changes {
+        let range = change
+            .range
+            .expect("`None` case already handled by finding the last full document change.");
+
+        // If the end of this change is at or past the start of the last change, then
+        // the `line_index` needed to apply this change is now invalid, so we have to
+        // rebuild it.
+        if range.end.line >= last_start_line {
+            line_index = biome_line_index::LineIndex::new(&contents);
+        }
+        last_start_line = range.start.line;
+
+        // This is a panic if we can't convert. It means we can't keep the document up
+        // to date and something is very wrong.
+        let range: std::ops::Range<usize> = from_proto::text_range(range, &line_index, encoding)
+            .expect("Can convert `range` from `Position` to `TextRange`.")
+            .into();
+
+        contents.replace_range(range, &change.text);
+    }
+
+    contents
+}
+
+#[cfg(test)]
+mod tests {
+    use biome_line_index::WideEncoding;
+
+    use super::*;
+
+    const ENCODING: PositionEncoding = PositionEncoding::Wide(WideEncoding::Utf16);
+
+    fn insert(text: &str, line: u32, character: u32) -> lsp_types::TextDocumentContentChangeEvent {
+        let position = lsp_types::Position::new(line, character);
+        lsp_types::TextDocumentContentChangeEvent {
+            range: Some(lsp_types::Range::new(position, position)),
+            range_length: None,
+            text: text.to_string(),
+        }
+    }
+
+    #[test]
+    fn test_apply_content_changes_incremental_inserts() {
+        // Type "lib" one character at a time, the way an editor streams it.
+        let after_l = apply_content_changes("", &[insert("l", 0, 0)], ENCODING);
+        assert_eq!(after_l, "l");
+
+        let after_i = apply_content_changes(&after_l, &[insert("i", 0, 1)], ENCODING);
+        assert_eq!(after_i, "li");
+
+        let after_b = apply_content_changes(&after_i, &[insert("b", 0, 2)], ENCODING);
+        assert_eq!(after_b, "lib");
+    }
+
+    #[test]
+    fn test_apply_content_changes_full_replacement_wins() {
+        // A range-less change replaces the whole buffer; earlier changes in the
+        // batch are discarded, later incremental ones apply on top of it.
+        let changes = vec![
+            insert("ignored", 0, 0),
+            lsp_types::TextDocumentContentChangeEvent {
+                range: None,
+                range_length: None,
+                text: "abc\n".to_string(),
+            },
+            insert("X", 0, 3),
+        ];
+        assert_eq!(apply_content_changes("old", &changes, ENCODING), "abcX\n");
+    }
+}

crates/ark/src/lsp/declarations.rs

@@ -89,68 +89,83 @@ fn ark_diagnostics_args<'tree>(ark_args: Node<'tree>, contents: &str) -> Option<
 mod test {
     use stdext::assert_match;
 
+    use crate::lsp::ark_file::test_ark_file;
     use crate::lsp::declarations::declare_ark_args;
     use crate::lsp::declarations::top_level_declare;
     use crate::lsp::declarations::top_level_declare_args;
-    use crate::lsp::document::Document;
 
     #[test]
     fn test_declare_args() {
-        let doc = Document::new("", None);
-        assert_match!(top_level_declare_args(&doc.ast, &doc.contents), None);
-
-        let doc = Document::new("declare()", None);
-        assert_match!(top_level_declare_args(&doc.ast, &doc.contents), Some(_));
-
-        let doc = Document::new("~declare()", None);
-        assert_match!(top_level_declare_args(&doc.ast, &doc.contents), Some(_));
-
-        let doc = Document::new("# foo\n#bar\n\ndeclare()", None);
-        assert_match!(top_level_declare_args(&doc.ast, &doc.contents), Some(_));
-
-        let doc = Document::new("# foo\nbar\n\ndeclare()", None);
-        assert_match!(top_level_declare_args(&doc.ast, &doc.contents), None);
+        let (db, file) = test_ark_file("");
+        assert_match!(
+            top_level_declare_args(file.tree_sitter(&db), file.contents(&db)),
+            None
+        );
+
+        let (db, file) = test_ark_file("declare()");
+        assert_match!(
+            top_level_declare_args(file.tree_sitter(&db), file.contents(&db)),
+            Some(_)
+        );
+
+        let (db, file) = test_ark_file("~declare()");
+        assert_match!(
+            top_level_declare_args(file.tree_sitter(&db), file.contents(&db)),
+            Some(_)
+        );
+
+        let (db, file) = test_ark_file("# foo\n#bar\n\ndeclare()");
+        assert_match!(
+            top_level_declare_args(file.tree_sitter(&db), file.contents(&db)),
+            Some(_)
+        );
+
+        let (db, file) = test_ark_file("# foo\nbar\n\ndeclare()");
+        assert_match!(
+            top_level_declare_args(file.tree_sitter(&db), file.contents(&db)),
+            None
+        );
     }
 
     #[test]
     fn test_declare_ark_args() {
-        let doc = Document::new("declare()", None);
-        let decls = top_level_declare_args(&doc.ast, &doc.contents).unwrap();
-        assert_match!(declare_ark_args(decls, &doc.contents), None);
+        let (db, file) = test_ark_file("declare()");
+        let decls = top_level_declare_args(file.tree_sitter(&db), file.contents(&db)).unwrap();
+        assert_match!(declare_ark_args(decls, file.contents(&db)), None);
 
-        let doc = Document::new("declare(ark())", None);
-        let decls = top_level_declare_args(&doc.ast, &doc.contents).unwrap();
-        assert_match!(declare_ark_args(decls, &doc.contents), Some(_));
+        let (db, file) = test_ark_file("declare(ark())");
+        let decls = top_level_declare_args(file.tree_sitter(&db), file.contents(&db)).unwrap();
+        assert_match!(declare_ark_args(decls, file.contents(&db)), Some(_));
 
-        let doc = Document::new("declare(foo, ark())", None);
-        let decls = top_level_declare_args(&doc.ast, &doc.contents).unwrap();
-        assert_match!(declare_ark_args(decls, &doc.contents), Some(_));
+        let (db, file) = test_ark_file("declare(foo, ark())");
+        let decls = top_level_declare_args(file.tree_sitter(&db), file.contents(&db)).unwrap();
+        assert_match!(declare_ark_args(decls, file.contents(&db)), Some(_));
     }
 
     #[test]
     fn test_declare_diagnostics() {
-        let doc = Document::new("", None);
-        let decls = top_level_declare(&doc.ast, &doc.contents);
+        let (db, file) = test_ark_file("");
+        let decls = top_level_declare(file.tree_sitter(&db), file.contents(&db));
         assert!(decls.diagnostics);
 
-        let doc = Document::new("declare(ark(diagnostics(enable = TRUE)))", None);
-        let decls = top_level_declare(&doc.ast, &doc.contents);
+        let (db, file) = test_ark_file("declare(ark(diagnostics(enable = TRUE)))");
+        let decls = top_level_declare(file.tree_sitter(&db), file.contents(&db));
         assert!(decls.diagnostics);
 
-        let doc = Document::new("declare(ark(diagnostics(enable = NULL)))", None);
-        let decls = top_level_declare(&doc.ast, &doc.contents);
+        let (db, file) = test_ark_file("declare(ark(diagnostics(enable = NULL)))");
+        let decls = top_level_declare(file.tree_sitter(&db), file.contents(&db));
         assert!(decls.diagnostics);
 
-        let doc = Document::new("declare(ark(diagnostics(enable = invalid())))", None);
-        let decls = top_level_declare(&doc.ast, &doc.contents);
+        let (db, file) = test_ark_file("declare(ark(diagnostics(enable = invalid())))");
+        let decls = top_level_declare(file.tree_sitter(&db), file.contents(&db));
         assert!(decls.diagnostics);
 
-        let doc = Document::new("~declare()", None);
-        let decls = top_level_declare(&doc.ast, &doc.contents);
+        let (db, file) = test_ark_file("~declare()");
+        let decls = top_level_declare(file.tree_sitter(&db), file.contents(&db));
         assert!(decls.diagnostics);
 
-        let doc = Document::new("declare(ark(diagnostics(enable = FALSE)))", None);
-        let decls = top_level_declare(&doc.ast, &doc.contents);
+        let (db, file) = test_ark_file("declare(ark(diagnostics(enable = FALSE)))");
+        let decls = top_level_declare(file.tree_sitter(&db), file.contents(&db));
         assert!(!decls.diagnostics);
     }
 }