(3) Extract out oak_srcref for srcref rebuilding (#1300)

Part of #1234
Branched from #1299 

Not much to see here, exact same srcref mechanism as before. Just
extracted into its own crate.

Author: DavisVaughan

Cargo.lock

@@ -87,6 +87,7 @@ oak_scan = { path = "crates/oak_scan" }
 oak_semantic = { path = "crates/oak_semantic" }
 oak_source = { path = "crates/oak_source" }
 oak_sources = { path = "crates/oak_sources" }
+oak_srcref = { path = "crates/oak_srcref" }
 once_cell = "1.21.4"
 parking_lot = "0.12.5"
 paste = "1.0.15"

Cargo.toml

@@ -1610,6 +1610,7 @@ foo
                 imports: vec![],
                 repository: None,
                 priority: None,
+                built: None,
                 fields: Dcf::new(),
             };
             let package = Package::from_parts(PathBuf::from("/mock/path"), description, namespace);
@@ -1705,6 +1706,7 @@ foo
                 imports: vec![],
                 repository: None,
                 priority: None,
+                built: None,
                 fields: Dcf::new(),
             };
             let package1 =
@@ -1723,6 +1725,7 @@ foo
                 imports: vec![],
                 repository: None,
                 priority: None,
+                built: None,
                 fields: Dcf::new(),
             };
             let package2 =
@@ -1782,6 +1785,7 @@ foo
                 imports: vec![],
                 repository: None,
                 priority: None,
+                built: None,
                 fields: Dcf::new(),
             };
             let package = Package::from_parts(PathBuf::from("/mock/path"), description, namespace);

crates/ark/src/lsp/diagnostics.rs

@@ -23,6 +23,7 @@ pub(crate) trait SourceHandler: Send + Sync {
 pub(crate) struct SourceRequest {
     name: String,
     version: String,
+    built: String,
     library_path: PathBuf,
 }
 
@@ -139,6 +140,13 @@ impl SourceRequest {
             ));
         };
 
+        let Some(built) = package.built(db).to_owned() else {
+            // Only ever runs on installed packages, which always carry a `Built:` field
+            return Err(anyhow::anyhow!(
+                "Package {name} is missing a `Built` field to provide sources for."
+            ));
+        };
+
         let library_path = match package.description_path(db) {
             FilePath::File(path) => {
                 match path.as_path().as_std_path().parent().and_then(Path::parent) {
@@ -160,6 +168,7 @@ impl SourceRequest {
         Ok(Self {
             name,
             version,
+            built,
             library_path,
         })
     }
@@ -176,6 +185,12 @@ impl SourceRequest {
         &self.version
     }
 
+    // TODO!: Remove when we have a production `SourceHandler`
+    #[cfg_attr(not(test), expect(dead_code))]
+    pub(crate) fn built(&self) -> &str {
+        &self.built
+    }
+
     // TODO!: Remove when we have a production `SourceHandler`
     #[cfg_attr(not(test), expect(dead_code))]
     pub(crate) fn library_path(&self) -> &Path {

crates/ark/src/lsp/sources.rs

@@ -165,6 +165,7 @@ async fn test_source_pipeline_ingests_package_sources() {
         assert_eq!(recorded.len(), 1);
         assert_eq!(recorded[0].name(), "donor");
         assert_eq!(recorded[0].version(), "0.0.0");
+        assert_eq!(recorded[0].built(), "dummy");
         assert_eq!(recorded[0].library_path(), lib.path());
     }
 

crates/ark/src/lsp/tests/sources.rs

@@ -46,7 +46,7 @@ pub(super) fn write_description(dir: &Path, name: &str) {
     std::fs::create_dir_all(dir).unwrap();
     std::fs::write(
         dir.join("DESCRIPTION"),
-        format!("Package: {name}\nVersion: 0.0.0\n"),
+        format!("Package: {name}\nVersion: 0.0.0\nBuilt: dummy\n"),
     )
     .unwrap();
 }

crates/ark/src/lsp/tests/utils.rs

@@ -140,6 +140,17 @@ impl Package {
             .map(|description| description.version.clone())
     }
 
+    /// The package's `Built:`, parsed lazily from `DESCRIPTION`. `None` when
+    /// the file is missing or has no `Built:` field (only installed packages have one).
+    ///
+    /// Build timestamps change on every install, so backdating probably isn't that
+    /// important here, so we don't track this method.
+    pub fn built(self, db: &dyn Db) -> Option<String> {
+        self.description(db)
+            .as_ref()
+            .and_then(|description| description.built.clone())
+    }
+
     /// The basename order from `DESCRIPTION`'s `Collate:` field, parsed
     /// lazily. `None` when the field (or the file) is absent. Narrow query
     /// over [`Package::description`], same backdating story as

crates/oak_db/src/package.rs

@@ -16,6 +16,9 @@ pub struct Description {
 
     pub priority: Option<Priority>,
 
+    /// Only present for installed packages
+    pub built: Option<String>,
+
     /// Raw DCF fields
     pub fields: Dcf,
 }
@@ -81,13 +84,16 @@ impl Description {
             None
         });
 
+        let built = fields.get("Built").map(str::to_string);
+
         Ok(Description {
             name,
             version,
             depends,
             imports,
             repository,
             priority,
+            built,
             fields,
         })
     }
@@ -232,6 +238,23 @@ Version: 1.0.0"#;
         assert_eq!(parsed.collate(), None);
     }
 
+    #[test]
+    fn parses_description_with_built() {
+        let desc = r#"Package: mypackage
+Version: 1.0.0
+Built: R 4.5.0; ; 2025-01-15 12:34:56 UTC; unix"#;
+        let parsed = Description::parse(desc).unwrap();
+        assert_eq!(
+            parsed.built.as_deref(),
+            Some("R 4.5.0; ; 2025-01-15 12:34:56 UTC; unix")
+        );
+
+        let desc = r#"Package: mypackage
+Version: 1.0.0"#;
+        let parsed = Description::parse(desc).unwrap();
+        assert_eq!(parsed.built, None);
+    }
+
     #[test]
     fn parses_description_with_unknown_repository() {
         let desc = r#"Package: mypackage

crates/oak_package_metadata/src/description.rs

@@ -0,0 +1,22 @@
+[package]
+name = "oak_srcref"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+hex.workspace = true
+log.workspace = true
+oak_cache.workspace = true
+oak_fs.workspace = true
+oak_r_process.workspace = true
+sha2.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true
+
+[lints]
+workspace = true

crates/oak_srcref/Cargo.toml

@@ -0,0 +1,104 @@
+# Extract source code from an installed R package using srcref metadata.
+#
+# See also `lobstr::src()`:
+# https://github.com/r-lib/lobstr/blob/85dd18d43e4612f98b05bb93019a7b12fb0bbf6b/R/src.R#L205
+#
+# Arguments (via commandArgs):
+#   1. package - Package name
+#   2. version - Expected package version
+#
+# Library paths have been set ahead of time via the `R_LIBS` environment
+# variable.
+#
+# On success: prints the concatenated source lines (with `#line` directives) to
+# stdout.
+#
+# On failure: exits with code 1 (package unexpectedly not installed, version
+# mismatch) or 2 (no srcrefs).
+
+args <- commandArgs(trailingOnly = TRUE)
+
+if (length(args) != 2L) {
+    message("'package' and 'version' are required arguments.")
+    quit(status = 1L)
+}
+
+package <- args[[1L]]
+version <- args[[2L]]
+
+# Make sure package is installed
+path <- find.package(package, quiet = TRUE)
+if (length(path) == 0L) {
+    message(paste0("Package '", package, "' is not installed"))
+    quit(status = 1L)
+}
+
+# Make sure installed version matches the requested version.
+# Normalize both to `package_version`s to handle versions like sf's 1.1-1.
+version <- as.character(package_version(version))
+installed_version <- as.character(packageVersion(package))
+if (installed_version != version) {
+    message(paste0(
+        "Version mismatch for '",
+        package,
+        "': ",
+        "installed ",
+        installed_version,
+        ", requested ",
+        version
+    ))
+    quit(status = 1L)
+}
+
+# Get the package namespace and find a function object.
+# Functions will contain the srcrefs if srcrefs have been kept, and are what we
+# can back out the full file structure from.
+ns <- asNamespace(package)
+exports <- getNamespaceExports(ns)
+
+fn <- NULL
+for (name in exports) {
+    candidate <- get0(name, envir = ns, mode = "function")
+    if (!is.null(candidate)) {
+        fn <- candidate
+        break
+    }
+}
+
+if (is.null(fn)) {
+    message(paste0("No functions found for package '", package, "'"))
+    quit(status = 2L)
+}
+
+extract_lines <- function(fn) {
+    srcref <- attr(fn, "srcref")
+    if (is.null(srcref)) {
+        return(NULL)
+    }
+
+    srcfile <- attr(srcref, "srcfile")
+    if (is.null(srcfile)) {
+        return(NULL)
+    }
+
+    original <- srcfile$original
+    if (is.null(original)) {
+        return(NULL)
+    }
+
+    lines <- original$lines
+    if (is.null(lines)) {
+        return(NULL)
+    }
+
+    lines
+}
+
+lines <- extract_lines(fn)
+
+if (is.null(lines)) {
+    message(paste0("No srcrefs found for package '", package, "'"))
+    quit(status = 2L)
+}
+
+cat(lines, sep = "\n")