Implement LibraryDefinitions backed by PackageCache

- Stored in `WorldState`
- Used in `goto_definition()`
- Split out `oak_library/` from `oak_package/` so that `oak_package` stays very lightweight
- Split out `oak_layers/` from `oak_index/`  so that `oak_index` is purely about the `SemanticIndex`, because `oak_layers` needs the heavier package cache utilities

Author: DavisVaughan

Cargo.lock

@@ -38,6 +38,7 @@ async-trait = "0.1.66"
 base64 = "0.21.0"
 biome_line_index = { git = "https://github.com/lionel-/biome", rev = "41d799cfa4cedd25625fc3f6bd7898532873f051" }
 biome_rowan = { git = "https://github.com/lionel-/biome", rev = "41d799cfa4cedd25625fc3f6bd7898532873f051" }
+biome_text_size = { git = "https://github.com/lionel-/biome", rev = "41d799cfa4cedd25625fc3f6bd7898532873f051" }
 blake3 = "1.8.2"
 bus = "2.3.0"
 cc = "1.1.22"
@@ -76,6 +77,8 @@ oak_fs = { path = "crates/oak_fs" }
 oak_ide = { path = "crates/oak_ide" }
 oak_index = { path = "crates/oak_index" }
 oak_index_vec = { path = "crates/oak_index_vec" }
+oak_layers = { path = "crates/oak_layers" }
+oak_library = { path = "crates/oak_library" }
 oak_package = { path = "crates/oak_package" }
 oak_r_process = { path = "crates/oak_r_process" }
 oak_sources = { path = "crates/oak_sources" }

Cargo.toml

@@ -47,6 +47,8 @@ notify.workspace = true
 oak_core.workspace = true
 oak_ide.workspace = true
 oak_index.workspace = true
+oak_layers.workspace = true
+oak_library.workspace = true
 oak_package.workspace = true
 once_cell.workspace = true
 regex.workspace = true
@@ -80,7 +82,7 @@ yaml-rust.workspace = true
 ark_test.workspace = true
 assert_matches.workspace = true
 insta.workspace = true
-oak_package = { workspace = true, features = ["testing"] }
+oak_library = { workspace = true, features = ["testing"] }
 stdext = { workspace = true, features = ["testing"] }
 tempfile.workspace = true
 

crates/ark/Cargo.toml

@@ -14,8 +14,8 @@ use anyhow::bail;
 use anyhow::Result;
 use harp::syntax::is_valid_symbol;
 use harp::syntax::sym_quote_invalid;
-use oak_package::library::Library;
-use oak_package::package::Package;
+use oak_library::library::Library;
+use oak_library::package::Package;
 use stdext::*;
 use tower_lsp::lsp_types::Diagnostic;
 use tower_lsp::lsp_types::DiagnosticSeverity;
@@ -169,14 +169,14 @@ pub(crate) fn generate_diagnostics(
     // If this is a package, add imported symbols to workspace
     if let Some(SourceRoot::Package(root)) = &state.root {
         // Add symbols from `importFrom()` directives
-        for import in &root.namespace.imports {
+        for import in &root.namespace().imports {
             context.workspace_symbols.insert(import.name.clone());
         }
 
         // Add symbols from `import()` directives
-        for package_import in &root.namespace.package_imports {
+        for package_import in &root.namespace().package_imports {
             if let Some(pkg) = state.library.get(package_import) {
-                for export in &pkg.namespace.exports {
+                for export in &pkg.namespace().exports {
                     context.workspace_symbols.insert(export.clone());
                 }
             }
@@ -193,7 +193,7 @@ pub(crate) fn generate_diagnostics(
     // test files setup.
     if testthat {
         if let Some(pkg) = state.library.get("testthat") {
-            for export in &pkg.namespace.exports {
+            for export in &pkg.namespace().exports {
                 context.workspace_symbols.insert(export.clone());
             }
         }
@@ -882,14 +882,14 @@ fn handle_package_attach_call(node: Node, context: &mut DiagnosticContext) -> an
     let package = insert_package_exports(package_name, attach_pos, context)?;
 
     // Also attach packages from `Depends` field
-    for package_name in package.description.depends.iter() {
+    for package_name in package.description().depends.iter() {
         insert_package_exports(package_name, attach_pos, context)?;
     }
 
     // Special handling for the tidyverse and tidymodels packages. Hard-coded
     // for now but in the future, this should probably be expressed as a
     // `DESCRIPTION` field like `Config/Needs/attach`.
-    let attach_field = match package.description.name.as_str() {
+    let attach_field = match package.description().name.as_str() {
         // https://github.com/tidyverse/tidyverse/blob/0231aafb/R/attach.R#L1
         "tidyverse" => {
             vec![
@@ -949,7 +949,7 @@ fn insert_package_exports(
         .library_symbols
         .entry(attach_pos)
         .or_default()
-        .extend(package.exported_symbols.iter().cloned());
+        .extend(package.exported_symbols().clone().into_vec());
 
     Ok(package)
 }
@@ -1132,12 +1132,13 @@ mod tests {
     use std::path::PathBuf;
 
     use harp::eval::RParseEvalOptions;
-    use oak_package::library::Library;
-    use oak_package::package::Package;
+    use oak_library::library::Library;
+    use oak_library::package::Package;
     use oak_package::package_description::Dcf;
     use oak_package::package_description::Description;
     use oak_package::package_namespace::Namespace;
     use once_cell::sync::Lazy;
+    use stdext::SortedVec;
     use tower_lsp::lsp_types;
     use tower_lsp::lsp_types::Position;
 
@@ -1638,7 +1639,7 @@ foo
         r_task(|| {
             // `mockpkg` exports `foo` and `bar`
             let namespace = Namespace {
-                exports: vec!["foo".to_string(), "bar".to_string()],
+                exports: SortedVec::from_vec(vec!["foo".to_string(), "bar".to_string()]),
                 imports: vec![],
                 package_imports: vec![],
             };
@@ -1736,7 +1737,7 @@ foo
         r_task(|| {
             // pkg1 exports `foo` and `bar`
             let namespace1 = Namespace {
-                exports: vec!["foo".to_string(), "bar".to_string()],
+                exports: SortedVec::from_vec(vec!["foo".to_string(), "bar".to_string()]),
                 imports: vec![],
                 package_imports: vec![],
             };
@@ -1753,7 +1754,7 @@ foo
 
             // pkg2 exports `bar` and `baz`
             let namespace2 = Namespace {
-                exports: vec!["bar".to_string(), "baz".to_string()],
+                exports: SortedVec::from_vec(vec!["bar".to_string(), "baz".to_string()]),
                 imports: vec![],
                 package_imports: vec![],
             };
@@ -1812,7 +1813,7 @@ foo
         r_task(|| {
             // `pkg` exports `foo` and `bar`
             let namespace = Namespace {
-                exports: vec!["foo".to_string(), "bar".to_string()],
+                exports: SortedVec::from_vec(vec!["foo".to_string(), "bar".to_string()]),
                 imports: vec![],
                 package_imports: vec![],
             };
@@ -1853,7 +1854,7 @@ foo
     #[test]
     fn test_penguins_symbol_no_diagnostic() {
         r_task(|| {
-            let palmerpenguins_dir = oak_package::package::temp_palmerpenguin();
+            let palmerpenguins_dir = oak_library::package::temp_palmerpenguin();
             let palmerpenguins_pkg = Package::load_from_folder(palmerpenguins_dir.path())
                 .unwrap()
                 .unwrap();

crates/ark/src/lsp/diagnostics.rs

@@ -427,7 +427,7 @@ fn new_syntax_diagnostic(
 
 #[cfg(test)]
 mod tests {
-    use oak_package::library::Library;
+    use oak_library::library::Library;
     use tower_lsp::lsp_types::Diagnostic;
     use tower_lsp::lsp_types::Position;
 

crates/ark/src/lsp/diagnostics_syntax.rs

@@ -33,6 +33,7 @@ pub(crate) fn goto_definition(
         &index,
         &state.external_scope(&uri),
         &state.library,
+        state.library_definitions.as_ref(),
     );
 
     if targets.is_empty() {
@@ -84,8 +85,8 @@ mod tests {
     use std::process::Command;
 
     use assert_matches::assert_matches;
-    use oak_package::library::Library;
-    use oak_package::package::Package;
+    use oak_library::library::Library;
+    use oak_library::package::Package;
     use oak_package::package_description::Description;
     use oak_package::package_namespace::Import;
     use oak_package::package_namespace::Namespace;

crates/ark/src/lsp/goto_definition.rs

@@ -5,7 +5,7 @@
 //
 //
 
-use oak_package::package::Package;
+use oak_library::package::Package;
 
 /// The root of a source tree.
 /// Currently only supports packages, but can be extended to scripts.

crates/ark/src/lsp/inputs/source_root.rs

@@ -18,7 +18,9 @@ use std::sync::RwLock;
 use anyhow::anyhow;
 use futures::stream::FuturesUnordered;
 use futures::StreamExt;
-use oak_package::library::Library;
+use oak_library::library::Library;
+use oak_library::library_definitions::LibraryDefinitions;
+use stdext::result::ResultExt;
 use tokio::sync::mpsc;
 use tokio::sync::mpsc::unbounded_channel as tokio_unbounded_channel;
 use tokio::task;
@@ -186,7 +188,7 @@ impl GlobalState {
     ///   and auxiliary loop.
     pub(crate) fn new(
         client: Client,
-        _r_home: PathBuf,
+        r_home: PathBuf,
         console_notification_tx: TokioUnboundedSender<ConsoleNotification>,
     ) -> Self {
         // Transmission channel for the main loop events. Shared with the
@@ -199,30 +201,36 @@ impl GlobalState {
             console_notification_tx,
         };
 
-        let mut state = Self {
-            world: WorldState::default(),
-            lsp_state,
-            client,
-            events_tx,
-            events_rx,
-        };
-
         // FIXME: We shouldn't call R code in the kernel to figure this out
-        if let Err(err) = crate::r_task(|| -> anyhow::Result<()> {
-            let paths: Vec<String> = harp::RFunction::new("base", ".libPaths")
+        let library_paths = crate::r_task(|| -> anyhow::Result<Vec<String>> {
+            Ok(harp::RFunction::new("base", ".libPaths")
                 .call()?
-                .try_into()?;
-
-            log::info!("Using library paths: {paths:#?}");
-            let paths: Vec<PathBuf> = paths.into_iter().map(PathBuf::from).collect();
-            state.world.library = Library::new(paths);
+                .try_into()?)
+        });
 
-            Ok(())
-        }) {
-            log::error!("Can't evaluate `libPaths()`: {err:?}");
+        let library_paths = match library_paths {
+            Ok(library_paths) => library_paths,
+            Err(err) => {
+                log::error!("Can't evaluate `libPaths()`: {err:?}");
+                Vec::new()
+            },
         };
 
-        state
+        let library_paths: Vec<PathBuf> = library_paths.into_iter().map(PathBuf::from).collect();
+
+        let library = Library::new(library_paths.clone());
+
+        let r = harp::command::r_executable(&r_home);
+        let library_definitions =
+            r.and_then(|r| LibraryDefinitions::new(r, library_paths).log_err());
+
+        Self {
+            world: WorldState::new(library, library_definitions),
+            lsp_state,
+            client,
+            events_tx,
+            events_rx,
+        }
     }
 
     /// Get `Event` transmission channel