(2) Extract out oak_source for CRAN downloads of R packages and R versions (#1299)

Part of #1234 
Branched from #1298 

Similar to the old cache model for CRAN sources, so not a ton to review.
The main structural differences are:

- For R packages, we have `get_cran()` and `insert_cran()` 
- For R versions, we have `get_r()` and `insert_r()`

Where both of these now download and keep _the whole thing_ rather than
trying to extract out only `R/`, which I think is going to pay off in
the long run.

Author: DavisVaughan

Cargo.lock

@@ -85,6 +85,7 @@ oak_package_metadata = { path = "crates/oak_package_metadata" }
 oak_r_process = { path = "crates/oak_r_process" }
 oak_scan = { path = "crates/oak_scan" }
 oak_semantic = { path = "crates/oak_semantic" }
+oak_source = { path = "crates/oak_source" }
 oak_sources = { path = "crates/oak_sources" }
 once_cell = "1.21.4"
 parking_lot = "0.12.5"

Cargo.toml

@@ -1 +1,2 @@
 pub mod file_lock;
+pub mod permissions;

crates/oak_fs/src/lib.rs

@@ -0,0 +1,8 @@
+use std::path::Path;
+
+/// Mark a file as read only
+pub fn set_readonly(path: &Path) -> std::io::Result<()> {
+    let mut permissions = std::fs::metadata(path)?.permissions();
+    permissions.set_readonly(true);
+    std::fs::set_permissions(path, permissions)
+}

crates/oak_fs/src/permissions.rs

@@ -0,0 +1,22 @@
+[package]
+name = "oak_source"
+version = "0.1.0"
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+flate2.workspace = true
+log.workspace = true
+oak_cache.workspace = true
+oak_fs.workspace = true
+tar.workspace = true
+ureq.workspace = true
+
+[dev-dependencies]
+tempfile.workspace = true
+
+[lints]
+workspace = true

crates/oak_source/Cargo.toml

@@ -0,0 +1,38 @@
+use std::path::Path;
+
+use crate::download::download_with_mirrors;
+use crate::download::Outcome;
+use crate::download::MIRRORS;
+use crate::extract;
+
+/// Download an R package's source tarball from CRAN and unpack it into `dir`
+///
+/// The tarball's top-level `{name}/` directory is stripped, so the package's files land
+/// directly under `dir` (e.g. `dir/R/`, `dir/DESCRIPTION`).
+///
+/// Returns `Ok(false)` if the package isn't on CRAN, which we treat as "source
+/// unavailable" rather than an error.
+pub(crate) fn populate(name: &str, version: &str, dir: &Path) -> anyhow::Result<bool> {
+    match download(name, version)? {
+        Outcome::Success(response) => {
+            extract::extract(response.into_body().into_reader(), dir)?;
+            Ok(true)
+        },
+        Outcome::NotFound => Ok(false),
+    }
+}
+
+fn download(name: &str, version: &str) -> anyhow::Result<Outcome> {
+    // Try released version
+    let outcome = download_with_mirrors(&format!("src/contrib/{name}_{version}.tar.gz"), MIRRORS)?;
+
+    if matches!(outcome, Outcome::Success(_)) {
+        return Ok(outcome);
+    }
+
+    // Try archive
+    download_with_mirrors(
+        &format!("src/contrib/Archive/{name}/{name}_{version}.tar.gz"),
+        MIRRORS,
+    )
+}

crates/oak_source/src/cran.rs

@@ -0,0 +1,59 @@
+use std::time::Duration;
+
+const HTTP_NOT_FOUND: u16 = 404;
+const HTTP_SERVICE_UNAVAILABLE: u16 = 503;
+
+const CONNECT_TIMEOUT: Duration = Duration::from_secs(10);
+const GLOBAL_TIMEOUT: Duration = Duration::from_secs(40);
+
+/// CRAN mirrors to try, in order
+pub(crate) const MIRRORS: &[&str] = &["https://cran.r-project.org", "https://cran.rstudio.com"];
+
+/// Outcome of a CRAN mirror HTTP request
+pub(crate) enum Outcome {
+    Success(ureq::http::Response<ureq::Body>),
+    NotFound,
+}
+
+pub(crate) fn download_with_mirrors(suffix: &str, mirrors: &[&str]) -> anyhow::Result<Outcome> {
+    if mirrors.is_empty() {
+        panic!("`mirrors` can't be empty.");
+    }
+
+    let mut last_error = None;
+
+    for mirror in mirrors {
+        let url = format!("{mirror}/{suffix}");
+
+        let request = ureq::get(&url)
+            .config()
+            .timeout_connect(Some(CONNECT_TIMEOUT))
+            .timeout_global(Some(GLOBAL_TIMEOUT))
+            .build();
+
+        match request.call() {
+            Ok(response) => return Ok(Outcome::Success(response)),
+
+            // Known to be not there, don't try any other mirrors
+            Err(ureq::Error::StatusCode(HTTP_NOT_FOUND)) => return Ok(Outcome::NotFound),
+
+            // Try next mirror, this one is temporarily unavailable
+            Err(ureq::Error::StatusCode(HTTP_SERVICE_UNAVAILABLE)) => {
+                last_error = Some(Err(ureq::Error::StatusCode(HTTP_SERVICE_UNAVAILABLE).into()));
+                continue;
+            },
+
+            // Try next mirror, this one timed out
+            Err(ureq::Error::Timeout(timeout)) => {
+                last_error = Some(Err(ureq::Error::Timeout(timeout).into()));
+                continue;
+            },
+
+            // Some unhandled error occurred, bail
+            Err(err) => return Err(err.into()),
+        };
+    }
+
+    // Every mirror returned `HTTP_SERVICE_UNAVAILABLE` or timed out
+    last_error.expect("`mirrors` was non-empty and we always set `last_error`")
+}

crates/oak_source/src/download.rs

@@ -0,0 +1,76 @@
+use std::collections::HashSet;
+use std::io::Read;
+use std::path::Component;
+use std::path::Path;
+use std::path::PathBuf;
+
+use flate2::read::GzDecoder;
+
+/// Unpack a gzipped tarball into `dir`, dropping its top-level directory and marking
+/// every file read only
+///
+/// CRAN package tarballs and R source tarballs both wrap their content in a single
+/// top-level directory (`{package}/` or `R-{version}/`). We strip it so the content lands
+/// directly under `dir`.
+///
+/// Files are marked as read only to discourage accidental edits.
+pub(crate) fn extract(reader: impl Read, dir: &Path) -> anyhow::Result<()> {
+    let gz = GzDecoder::new(reader);
+    let mut archive = tar::Archive::new(gz);
+
+    // Parent directories we've already created
+    let mut created: HashSet<PathBuf> = HashSet::new();
+
+    for entry in archive.entries()? {
+        let mut entry = entry?;
+        let is_file = entry.header().entry_type().is_file();
+
+        let path = entry.path()?.into_owned();
+        let Some(relative) = strip_top_level(&path) else {
+            // The top-level directory entry itself, or an unsafe path, nothing to unpack
+            continue;
+        };
+
+        let destination = dir.join(relative);
+
+        // We must create parent directories before unpacking into them. We remember ones
+        // we've already created to avoid thousands of redundant `create_dir_all()` calls.
+        if let Some(parent) = destination.parent() {
+            if !created.contains(parent) {
+                std::fs::create_dir_all(parent)?;
+                created.insert(parent.to_path_buf());
+            }
+        }
+
+        entry.unpack(&destination)?;
+
+        if is_file {
+            oak_fs::permissions::set_readonly(&destination)?;
+        }
+    }
+
+    Ok(())
+}
+
+/// Strip the single top-level directory from a tarball entry path
+///
+/// Returns `None` for the top-level directory entry itself, or for any unsafe path
+/// (absolute, or containing `..`) that could escape the destination.
+fn strip_top_level(path: &Path) -> Option<&Path> {
+    let mut components = path.components();
+    components.next()?;
+
+    let rest = components.as_path();
+
+    if rest.as_os_str().is_empty() {
+        // The top-level directory entry itself
+        return None;
+    }
+
+    if !rest.components().all(|c| matches!(c, Component::Normal(_))) {
+        // Something would be strange here!
+        return None;
+    }
+
+    Some(rest)
+}