(Sources-3) oak_scan::set_package_sources() (#1288)

Progress towards #1234
Branched from #1287

This PR teaches `oak_scan` how to update an installed `Package`'s
`files` and `scripts` from an _external on disk directory_.

The idea is that some external source provider will return a flat
`directory` of R files that correspond to the source files for a
`Package`. The new `oak_scan::set_package_sources()` helper iterates
over that `directory`, collecting the R files (but not reading them!),
sorting and splitting them in accordance with `collate`, and stashes
them in `files` and `scripts` of the `Package`.

> [!CAUTION]
> This PR has an important side effect. Since `package.files()` now live
in a totally separate directory outside any `LiveRoot`, the
`File::package` backpointer is now a load bearing feature.
>
> Consider `File::root()`. It switches on `self.package()`. If there is
no package backpointer, it would go through either:
> - `root_by_file()`, which looks inside `LiveRoot::Workspace` and
`LiveRoot::Library` root paths for the file's `path`, neither of which
would hold it
> - `root_by_path()`, which is for orphaned files, and also uses paths
contained within `WorkspaceRoots`
>
> So we need to instead detect `self.package()` and go through
`root_by_package()`, which doesn't do a `path` check, it just detects
that the `Package` backpointer is contained within a `LiveRoot` (by
equality, not by `path` containment). We already do all this, it is just
important to point it out now.
>
> `File::imports()` is similar

Author: DavisVaughan

crates/oak_scan/src/inputs.rs

@@ -18,6 +18,7 @@
 
 use std::collections::HashSet;
 use std::hash::Hash;
+use std::path::Path;
 use std::path::PathBuf;
 
 use aether_path::FilePath;
@@ -30,6 +31,7 @@ use oak_db::Root;
 use salsa::Setter;
 
 use crate::lookup::package_by_path;
+use crate::packages::read_package_sources;
 use crate::stale::remove_from_stale_files;
 use crate::stale::remove_from_stale_packages;
 use crate::stale::stale_file_by_path;
@@ -113,6 +115,14 @@ pub trait DbScan: Db + DbInputs {
     /// If the file is in a live workspace / library container, the call is a
     /// no-op.
     fn close_editor(&mut self, path: &FilePath);
+
+    /// Set `package`'s `files` / `scripts` to the `.R` files found directly
+    /// under `directory`, respecting the package's `Collate` rules.
+    ///
+    /// Used to ingest sources produced by an external tool into an already-registered
+    /// library `Package` whose `files` start empty (the scanner registers installed
+    /// packages without sources).
+    fn set_package_sources(&mut self, package: Package, directory: &Path);
 }
 
 impl<DB: Db + DbInputs> DbScan for DB {
@@ -165,6 +175,23 @@ impl<DB: Db + DbInputs> DbScan for DB {
             stale.set_files(self).to(stale_files);
         }
     }
+
+    fn set_package_sources(&mut self, package: Package, directory: &Path) {
+        let (files, scripts) = read_package_sources(directory, package.collation(self).as_deref());
+
+        let files: Vec<File> = files
+            .into_iter()
+            .map(|entry| upsert_root_file(self, Some(package), entry))
+            .collect();
+
+        let scripts: Vec<File> = scripts
+            .into_iter()
+            .map(|entry| upsert_root_file(self, Some(package), entry))
+            .collect();
+
+        package.set_files(self).to(files);
+        package.set_scripts(self).to(scripts);
+    }
 }
 
 /// Extension methods on [`Root`] for placement-aware updates.

crates/oak_scan/src/packages.rs

@@ -210,6 +210,46 @@ fn read_workspace_package(package_dir: &Path) -> Option<PackageEntry> {
     Some(package)
 }
 
+/// Walk `directory`, collecting `.R` files
+///
+/// `.R` files are split into either `files` or `scripts` based on `collation`,
+/// matching [read_workspace_package()].
+///
+/// The `directory` is not walked recursively. We expect that this is a flat directory of
+/// `.R` files.
+pub(crate) fn read_package_sources(
+    directory: &Path,
+    collation: Option<&[String]>,
+) -> (Vec<FileEntry>, Vec<FileEntry>) {
+    let Ok(entries) = fs::read_dir(directory) else {
+        log::warn!("Cannot read sources directory: {}", directory.display());
+        return (Vec::new(), Vec::new());
+    };
+
+    let mut files: Vec<(PathBuf, FileEntry)> = Vec::new();
+
+    for entry in entries.flatten() {
+        let path = entry.path();
+        if !is_r_file(&path) {
+            continue;
+        }
+        let Some(file_path) = FilePath::from_path_buf(path.clone()) else {
+            log::warn!("Skipping R file, can't build a URL: {}", path.display());
+            continue;
+        };
+        let revision = file_revision(&path);
+        files.push((path, FileEntry {
+            path: file_path,
+            revision,
+        }));
+    }
+
+    match collation {
+        Some(order) => order_by_collation(files, order),
+        None => order_alphabetically(files),
+    }
+}
+
 /// Split the package's `R/*.R` files into `(loadable, leftover)` using the
 /// `Collate:` directive: `loadable` is the listed files in that order, and
 /// `leftover` is the R/ files not listed. Logs mismatches in either direction:

crates/oak_scan/src/tests.rs

@@ -1,4 +1,5 @@
 mod scheduler;
+mod sources;
 mod stale;
 mod watch;
 mod workspace;

crates/oak_scan/src/tests/sources.rs

@@ -0,0 +1,116 @@
+use std::fs;
+use std::path::Path;
+
+use oak_db::Db;
+use oak_db::File;
+use oak_db::OakDatabase;
+
+use crate::DbScan;
+
+/// Write an installed package's `DESCRIPTION` under `lib`
+fn write_description(lib: &Path, name: &str, collate: Option<&[&str]>) {
+    let directory = lib.join(name);
+    fs::create_dir_all(&directory).unwrap();
+
+    let mut description = format!("Package: {name}\nVersion: 1.0.0\n");
+
+    if let Some(collate) = collate {
+        description.push_str("Collate:");
+        for basename in collate {
+            description.push_str(&format!(" '{basename}'"));
+        }
+        description.push('\n');
+    }
+
+    fs::write(directory.join("DESCRIPTION"), description).unwrap();
+}
+
+/// Write `*.R` files directly under `directory`, the layout a source provider hands to
+/// `set_package_sources`.
+fn write_source_directory(directory: &Path, r_files: &[(&str, &str)]) {
+    fs::create_dir_all(directory).unwrap();
+    for (basename, contents) in r_files {
+        fs::write(directory.join(basename), contents).unwrap();
+    }
+}
+
+fn basenames(db: &OakDatabase, files: &[File]) -> Vec<String> {
+    files
+        .iter()
+        .map(|file| {
+            file.path(db)
+                .to_url()
+                .path()
+                .rsplit('/')
+                .next()
+                .unwrap()
+                .to_string()
+        })
+        .collect()
+}
+
+#[test]
+fn test_set_package_sources_alphabetical() {
+    let lib = tempfile::tempdir().unwrap();
+    write_description(lib.path(), "mypkg", None);
+
+    let mut db = OakDatabase::new();
+    db.set_library_paths(&[lib.path().to_path_buf()]);
+    let pkg = db.package_by_name("mypkg").unwrap();
+    assert!(pkg.files(&db).is_empty());
+
+    let src = tempfile::tempdir().unwrap();
+    write_source_directory(src.path(), &[("b.R", "b <- 1\n"), ("a.R", "a <- 1\n")]);
+    db.set_package_sources(pkg, src.path());
+
+    let files = pkg.files(&db).clone();
+    assert_eq!(basenames(&db, &files), ["a.R", "b.R"]);
+    assert!(pkg.scripts(&db).is_empty());
+
+    // Each file carries the package backpointer (path-based containment can't
+    // reach a cache dir outside the library root, so resolution relies on it).
+    for file in &files {
+        assert_eq!(file.package(&db), Some(pkg));
+    }
+
+    // Source is read lazily from disk via the revision, not an editor override.
+    assert_eq!(files[0].source_text(&db), "a <- 1\n");
+}
+
+#[test]
+fn test_set_package_sources_collation_order() {
+    let lib = tempfile::tempdir().unwrap();
+    write_description(lib.path(), "mypkg", Some(&["b.R", "a.R"]));
+
+    let mut db = OakDatabase::new();
+    db.set_library_paths(&[lib.path().to_path_buf()]);
+    let pkg = db.package_by_name("mypkg").unwrap();
+    assert!(pkg.files(&db).is_empty());
+
+    let src = tempfile::tempdir().unwrap();
+    write_source_directory(src.path(), &[("a.R", "a <- 1\n"), ("b.R", "b <- 1\n")]);
+    db.set_package_sources(pkg, src.path());
+
+    // Collation order preserved!
+    assert_eq!(basenames(&db, &pkg.files(&db).clone()), ["b.R", "a.R"]);
+    assert!(pkg.scripts(&db).is_empty());
+}
+
+#[test]
+fn test_set_package_sources_leftover_becomes_scripts() {
+    let lib = tempfile::tempdir().unwrap();
+    write_description(lib.path(), "mypkg", Some(&["a.R"]));
+
+    let mut db = OakDatabase::new();
+    db.set_library_paths(&[lib.path().to_path_buf()]);
+    let pkg = db.package_by_name("mypkg").unwrap();
+    assert!(pkg.files(&db).is_empty());
+
+    let src = tempfile::tempdir().unwrap();
+    write_source_directory(src.path(), &[("a.R", "a <- 1\n"), ("c.R", "c <- 1\n")]);
+    db.set_package_sources(pkg, src.path());
+
+    // `c.R` isn't in `Collate:`, so R won't load it. It ends up in `scripts`.
+    assert_eq!(basenames(&db, &pkg.files(&db).clone()), ["a.R"]);
+    assert_eq!(basenames(&db, &pkg.scripts(&db).clone()), ["c.R"]);
+}