Adjustments to oak_sources/

- Include `INDEX` in the cache
- Add `PackageCache` trait
- Add `struct TestPackageCache` with testing feature

Author: DavisVaughan

crates/oak_sources/Cargo.toml

@@ -20,8 +20,14 @@ serde.workspace = true
 serde_json.workspace = true
 sha2.workspace = true
 tar.workspace = true
-tempfile.workspace = true
+tempfile = { workspace = true, optional = true }
 ureq.workspace = true
 
+[dev-dependencies]
+tempfile.workspace = true
+
+[features]
+testing = ["dep:tempfile"]
+
 [lints]
 workspace = true

crates/oak_sources/src/installed_package.rs

@@ -13,12 +13,15 @@ pub(crate) struct InstalledPackage {
 }
 
 impl InstalledPackage {
-    pub(crate) fn find(package: &str, library_paths: &[PathBuf]) -> anyhow::Result<Option<Self>> {
+    pub(crate) fn find<P: AsRef<Path>>(
+        package: &str,
+        library_paths: &[P],
+    ) -> anyhow::Result<Option<Self>> {
         let mut library_path = None;
 
         for library_path_candidate in library_paths {
-            if library_path_candidate.join(package).exists() {
-                library_path = Some(library_path_candidate);
+            if library_path_candidate.as_ref().join(package).exists() {
+                library_path = Some(library_path_candidate.as_ref());
                 break;
             }
         }
@@ -51,7 +54,7 @@ impl InstalledPackage {
         Ok(Some(Self {
             key,
             name: package.to_string(),
-            library_path: library_path.clone(),
+            library_path: library_path.to_path_buf(),
             description,
             description_hash,
         }))
@@ -92,6 +95,10 @@ impl InstalledPackage {
         self.package_path().join("NAMESPACE")
     }
 
+    pub(crate) fn index_path(&self) -> PathBuf {
+        self.package_path().join("INDEX")
+    }
+
     pub(crate) fn description_hash(&self) -> &str {
         &self.description_hash
     }

crates/oak_sources/src/lib.rs

@@ -5,8 +5,12 @@ mod fs;
 mod hash;
 mod installed_package;
 mod srcref;
+#[cfg(any(test, feature = "testing"))]
+pub mod test;
+pub mod traits;
 
 use std::collections::HashSet;
+use std::path::Path;
 use std::path::PathBuf;
 use std::sync::RwLock;
 
@@ -35,6 +39,12 @@ const METADATA_FILENAME: &str = ".metadata";
 /// the DESCRIPTION `Build:` timestamp being different).
 const ONE_WEEK: TimeDelta = TimeDelta::weeks(1);
 
+impl crate::traits::PackageCache for PackageCache {
+    fn get(&self, package: &str) -> Option<PathBuf> {
+        self.get(package)
+    }
+}
+
 /// A cache of extracted R package sources
 ///
 /// # On disk layout
@@ -109,12 +119,13 @@ const ONE_WEEK: TimeDelta = TimeDelta::weeks(1);
 ///
 /// [`get`]: PackageCache::get
 /// [`clean`]: PackageCache::clean
+#[derive(Debug)]
 pub struct PackageCache {
-    /// Path to `R` binary
+    /// Path to an R executable
     r: PathBuf,
 
-    /// Set of R library paths
-    r_libpaths: Vec<PathBuf>,
+    /// Library paths to consider
+    library_paths: Vec<PathBuf>,
 
     /// On disk cache directory root
     cache_root: file_lock::Filesystem,
@@ -148,7 +159,7 @@ struct Metadata {
 }
 
 impl PackageCache {
-    pub fn new(r: PathBuf, r_libpaths: Vec<PathBuf>) -> anyhow::Result<Self> {
+    pub fn new(r: PathBuf, library_paths: Vec<PathBuf>) -> anyhow::Result<Self> {
         let cache_root = file_lock::Filesystem::new(crate::fs::sources_dir()?);
         cache_root.create_dir()?;
 
@@ -165,7 +176,7 @@ impl PackageCache {
 
         Ok(Self {
             r,
-            r_libpaths,
+            library_paths,
             cache_root,
             cache_root_lock,
             source_unavailable: RwLock::new(HashSet::new()),
@@ -188,7 +199,7 @@ impl PackageCache {
     }
 
     fn get_result(&self, package: &str) -> anyhow::Result<Option<PathBuf>> {
-        let Some(package) = InstalledPackage::find(package, &self.r_libpaths)? else {
+        let Some(package) = InstalledPackage::find(package, &self.library_paths)? else {
             // Not even installed
             return Ok(None);
         };
@@ -212,9 +223,9 @@ impl PackageCache {
         // Write path
         let result = if matches!(package.description().priority, Some(Priority::Base)) {
             // R version to download is the same as the base package version
-            self.try_populate_base(&package.description().version)
+            self.try_populate_base(&package.description().version, &self.library_paths)
         } else {
-            self.try_populate(&package)
+            self.try_populate(&package, &self.r, &self.library_paths)
         };
 
         match result {
@@ -245,7 +256,11 @@ impl PackageCache {
         }
     }
 
-    fn try_populate_base(&self, version: &str) -> anyhow::Result<bool> {
+    fn try_populate_base<P: AsRef<Path>>(
+        &self,
+        version: &str,
+        library_paths: &[P],
+    ) -> anyhow::Result<bool> {
         // Download the R sources in their entirety
         let Some(bytes) = crate::base::download(version)? else {
             log::trace!("No R source tarball on CRAN for version {version}");
@@ -254,7 +269,7 @@ impl PackageCache {
 
         // Populate all base packages from the download
         for package in crate::base::BASE_PACKAGES {
-            let Some(package) = InstalledPackage::find(package, &self.r_libpaths)? else {
+            let Some(package) = InstalledPackage::find(package, library_paths)? else {
                 // It would be very odd to not find a base package
                 return Ok(false);
             };
@@ -304,14 +319,24 @@ impl PackageCache {
             )?;
         }
 
+        crate::fs::copy_as_readonly(
+            package.index_path(),
+            destination_lock.parent().join("INDEX"),
+        )?;
+
         // Last! `.metadata` is the completion sentinel.
         self.write_metadata(package, &destination_lock)?;
 
         Ok(())
     }
 
     /// Writes `DESCRIPTION`, `NAMESPACE`, and `R/` to the cache entry, if possible
-    fn try_populate(&self, package: &InstalledPackage) -> anyhow::Result<bool> {
+    fn try_populate<P: AsRef<Path>, Q: AsRef<Path>>(
+        &self,
+        package: &InstalledPackage,
+        r: P,
+        library_paths: &[Q],
+    ) -> anyhow::Result<bool> {
         // Take per-key exclusive lock
         let destination = self.cache_root.join(package.key());
         destination.create_dir()?;
@@ -326,7 +351,7 @@ impl PackageCache {
         // writing `.metadata`.
         destination_lock.remove_siblings()?;
 
-        if !self.write_r_files(package, &destination_lock)? {
+        if !self.write_r_files(package, r, library_paths, &destination_lock)? {
             return Ok(false);
         }
 
@@ -338,25 +363,31 @@ impl PackageCache {
             package.namespace_path(),
             destination_lock.parent().join("NAMESPACE"),
         )?;
+        crate::fs::copy_as_readonly(
+            package.index_path(),
+            destination_lock.parent().join("INDEX"),
+        )?;
 
         // Last! Only write `.metadata` if all other writes succeed. It is our completion sentinal.
         self.write_metadata(package, &destination_lock)?;
 
         Ok(true)
     }
 
-    fn write_r_files(
+    fn write_r_files<P: AsRef<Path>, Q: AsRef<Path>>(
         &self,
         package: &InstalledPackage,
+        r: P,
+        library_paths: &[Q],
         destination_lock: &FileLock,
     ) -> anyhow::Result<bool> {
         // Try caching from srcref
         match crate::srcref::cache_srcref(
             package.name(),
             &package.description().version,
             destination_lock,
-            &self.r,
-            &self.r_libpaths,
+            r,
+            library_paths,
         ) {
             Ok(true) => {
                 log::trace!(
@@ -515,22 +546,3 @@ impl PackageCache {
         Ok(())
     }
 }
-
-// // For local testing
-// #[cfg(test)]
-// mod tests {
-//     use std::path::PathBuf;
-//
-//     use crate::PackageCache;
-//
-//     #[test]
-//     fn testit() {
-//         let r_script_path = PathBuf::from("/usr/local/bin/Rscript");
-//         let r_libpaths = vec![
-//             PathBuf::from("/Users/davis/Library/R/arm64/4.5/library"),
-//             PathBuf::from("/Library/Frameworks/R.framework/Versions/4.5-arm64/Resources/library"),
-//         ];
-//         let cache = PackageCache::new(r_script_path, r_libpaths).unwrap();
-//         cache.get("utils");
-//     }
-// }

crates/oak_sources/src/srcref.rs

@@ -1,5 +1,4 @@
 use std::path::Path;
-use std::path::PathBuf;
 
 use oak_fs::file_lock::FileLock;
 
@@ -9,25 +8,25 @@ const SCRIPT: &str = include_str!("../scripts/srcrefs.R");
 /// them to the cache at the parent folder containing `destination_lock`
 ///
 /// Launches a sidecar R session to read the srcrefs from the installed package.
-pub(crate) fn cache_srcref(
+pub(crate) fn cache_srcref<P: AsRef<Path>, Q: AsRef<Path>>(
     package: &str,
     version: &str,
     destination_lock: &FileLock,
-    r: &Path,
-    r_libpaths: &[PathBuf],
+    r: P,
+    library_paths: &[Q],
 ) -> anyhow::Result<bool> {
     let args = &[package, version];
 
     // Set `R_LIBS` to ensure the correct R package libraries are checked
     // `:` on Unix, `;` on Windows, see `?R_LIBS`
-    let libpaths = r_libpaths
+    let library_paths = library_paths
         .iter()
-        .map(|libpath| libpath.to_string_lossy())
+        .map(|library_path| library_path.as_ref().to_string_lossy())
         .collect::<Vec<_>>()
         .join(if cfg!(windows) { ";" } else { ":" });
-    let env = &[("R_LIBS", libpaths.as_str())];
+    let env = &[("R_LIBS", library_paths.as_str())];
 
-    let output = oak_r_process::run_text(r, SCRIPT, args, env)?;
+    let output = oak_r_process::run_text(r.as_ref(), SCRIPT, args, env)?;
 
     let code = output.status.code().unwrap_or(1);
 
@@ -120,6 +119,8 @@ fn parse_line_directive(line: &str) -> Option<String> {
 
 #[cfg(test)]
 mod tests {
+    use std::path::PathBuf;
+
     use oak_fs::file_lock::Filesystem;
 
     use super::*;

crates/oak_sources/src/test.rs

@@ -0,0 +1,48 @@
+use std::path::Path;
+use std::path::PathBuf;
+
+use tempfile::TempDir;
+
+impl crate::traits::PackageCache for TestPackageCache {
+    fn get(&self, package: &str) -> Option<PathBuf> {
+        self.get(package)
+    }
+}
+
+/// A fake package cache that can be used for testing
+#[derive(Debug)]
+pub struct TestPackageCache {
+    root: TempDir,
+}
+
+impl TestPackageCache {
+    pub fn new() -> anyhow::Result<Self> {
+        let root = TempDir::new()?;
+        Ok(Self { root })
+    }
+
+    pub fn get(&self, package: &str) -> Option<PathBuf> {
+        let package = self.root.path().join(package);
+
+        if package.exists() {
+            Some(package)
+        } else {
+            None
+        }
+    }
+
+    pub fn add(&self, package: &str, files: Vec<(&Path, &str)>) -> anyhow::Result<()> {
+        let package = self.root.path().join(package);
+        std::fs::create_dir(&package)?;
+
+        let r = package.join("R");
+        std::fs::create_dir(&r)?;
+
+        for (name, content) in files {
+            let path = r.join(name);
+            std::fs::write(path, content)?;
+        }
+
+        Ok(())
+    }
+}

crates/oak_sources/src/traits.rs

@@ -0,0 +1,10 @@
+use std::path::PathBuf;
+
+/// Trait for the public API of any package cache
+///
+/// Implemented by the main [crate::PackageCache] itself, but also by
+/// [crate::test::TestPackageCache] so that you can generate a test cache that doesn't
+/// need internet access or access to a live R session.
+pub trait PackageCache: std::fmt::Debug + Sync + Send {
+    fn get(&self, package: &str) -> Option<PathBuf>;
+}