From 449f693f96bdf292273aa9f5fcc0ea502b3631e6 Mon Sep 17 00:00:00 2001 From: Harsh Sahu Date: Sat, 21 Mar 2026 02:08:27 +0530 Subject: [PATCH] feat(cli): prototype publish/archive contract for Dora packages --- Cargo.lock | 73 ++++++++- binaries/cli/Cargo.toml | 8 + binaries/cli/src/lib.rs | 2 + binaries/cli/src/package_archive.rs | 224 +++++++++++++++++++++++++++ binaries/cli/src/publish_metadata.rs | 199 ++++++++++++++++++++++++ 5 files changed, 502 insertions(+), 4 deletions(-) create mode 100644 binaries/cli/src/package_archive.rs create mode 100644 binaries/cli/src/publish_metadata.rs diff --git a/Cargo.lock b/Cargo.lock index 51b31d0aa..5737e39b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1556,8 +1556,10 @@ dependencies = [ "duration-str", "env_logger", "eyre", + "flate2", "futures", "git2", + "hex", "inquire 0.5.3", "itertools 0.14.0", "log", @@ -1571,11 +1573,15 @@ dependencies = [ "serde", "serde_json", "serde_yaml 0.9.34+deprecated", + "sha2", "sysinfo 0.36.1", "tabwriter", + "tar", + "tempfile", "termcolor", "tokio", "tokio-stream", + "toml", "tracing", "tracing-log", "uuid", @@ -4672,7 +4678,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit", + "toml_edit 0.25.5+spec-1.1.0", ] [[package]] @@ -5827,6 +5833,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -6647,6 +6662,27 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + [[package]] name = "toml_datetime" version = "1.0.1+spec-1.1.0" @@ -6656,6 +6692,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.13.0", + "serde", + "serde_spanned", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_edit" version = "0.25.5+spec-1.1.0" @@ -6663,9 +6713,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ca1a40644a28bce036923f6a431df0b34236949d111cc07cb6dca830c9ef2e1" dependencies = [ "indexmap 2.13.0", - "toml_datetime", + "toml_datetime 1.0.1+spec-1.1.0", "toml_parser", - "winnow", + "winnow 1.0.0", ] [[package]] @@ -6674,9 +6724,15 @@ version = "1.0.10+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" dependencies = [ - "winnow", + "winnow 1.0.0", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tonic" version = "0.14.5" @@ -7890,6 +7946,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.0" diff --git a/binaries/cli/Cargo.toml b/binaries/cli/Cargo.toml index 97fd3a855..fd7d13dc8 100644 --- a/binaries/cli/Cargo.toml +++ b/binaries/cli/Cargo.toml @@ -35,6 +35,7 @@ serde = { version = "1.0.136", features = ["derive"] } serde_yaml = { workspace = true } webbrowser = "0.8.3" serde_json = "1.0.86" +toml = "0.8.19" termcolor = "1.1.3" uuid = { version = "1.7", features = ["v7", "serde"] } inquire = "0.5.2" @@ -77,10 +78,17 @@ git2 = { workspace = true } zenoh = { workspace = true } arrow-json.workspace = true chrono = "0.4.42" +flate2 = "1.1.2" +tar = "0.4.44" +sha2 = "0.10.9" +hex = "0.4.3" [build-dependencies] pyo3-build-config = "0.23" +[dev-dependencies] +tempfile = "3.23.0" + [lib] name = "dora_cli" path = "src/lib.rs" diff --git a/binaries/cli/src/lib.rs b/binaries/cli/src/lib.rs index e6fa41aea..738925035 100644 --- a/binaries/cli/src/lib.rs +++ b/binaries/cli/src/lib.rs @@ -8,6 +8,8 @@ mod command; mod common; mod formatting; pub mod output; +pub mod package_archive; +pub mod publish_metadata; pub mod session; mod template; diff --git a/binaries/cli/src/package_archive.rs b/binaries/cli/src/package_archive.rs new file mode 100644 index 000000000..566dec8bf --- /dev/null +++ b/binaries/cli/src/package_archive.rs @@ -0,0 +1,224 @@ +use crate::publish_metadata::{PublishManifest, PublishedPackageRecord}; +use eyre::{Context, bail}; +use flate2::{Compression, read::GzDecoder, write::GzEncoder}; +use sha2::{Digest, Sha256}; +use std::{ + io::{Cursor, Read}, + path::Path, +}; +use tar::{Archive, Builder}; + +pub fn build_published_package( + package_dir: &Path, +) -> eyre::Result<(PublishedPackageRecord, Vec)> { + let manifest_path = package_dir.join("Dora.toml"); + let manifest = PublishManifest::from_dora_toml_path(&manifest_path)?; + let archive = create_package_archive(package_dir)?; + let checksum = archive_checksum(&archive); + Ok(( + PublishedPackageRecord::from_manifest(manifest, checksum), + archive, + )) +} + +pub fn create_package_archive(package_dir: &Path) -> eyre::Result> { + if !package_dir.is_dir() { + bail!( + "package directory `{}` does not exist", + package_dir.display() + ); + } + + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + { + let mut builder = Builder::new(&mut encoder); + append_directory(&mut builder, package_dir, package_dir)?; + builder.finish()?; + } + encoder.finish().map_err(Into::into) +} + +pub fn archive_checksum(archive_bytes: &[u8]) -> String { + let checksum = Sha256::digest(archive_bytes); + format!("sha256:{}", hex::encode(checksum)) +} + +pub fn validate_published_archive( + record: &PublishedPackageRecord, + archive_bytes: &[u8], +) -> eyre::Result<()> { + let actual_checksum = archive_checksum(archive_bytes); + if actual_checksum != record.checksum { + bail!( + "archive checksum mismatch for `{}@{}`: expected `{}`, got `{}`", + record.name, + record.version, + record.checksum, + actual_checksum + ); + } + + let manifest = read_manifest_from_archive(archive_bytes)?; + if manifest.name != record.name { + bail!( + "archive manifest name `{}` does not match published metadata `{}`", + manifest.name, + record.name + ); + } + if manifest.version != record.version { + bail!( + "archive manifest version `{}` does not match published metadata `{}`", + manifest.version, + record.version + ); + } + if manifest.dependencies != record.dependencies { + bail!( + "archive manifest dependencies do not match published metadata for `{}@{}`", + record.name, + record.version + ); + } + + Ok(()) +} + +fn append_directory( + builder: &mut Builder<&mut GzEncoder>>, + root: &Path, + current: &Path, +) -> eyre::Result<()> { + for entry in std::fs::read_dir(current) + .with_context(|| format!("failed to read directory `{}`", current.display()))? + { + let entry = entry?; + let path = entry.path(); + let relative = path.strip_prefix(root).unwrap(); + + if entry.file_type()?.is_dir() { + builder.append_dir(relative, &path)?; + append_directory(builder, root, &path)?; + } else if entry.file_type()?.is_file() { + let mut file = std::fs::File::open(&path) + .with_context(|| format!("failed to open file `{}`", path.display()))?; + builder + .append_file(relative, &mut file) + .with_context(|| format!("failed to archive file `{}`", path.display()))?; + } + } + + Ok(()) +} + +fn read_manifest_from_archive(archive_bytes: &[u8]) -> eyre::Result { + let gz = GzDecoder::new(Cursor::new(archive_bytes)); + let mut archive = Archive::new(gz); + + for entry in archive.entries()? { + let mut entry = entry?; + let path = entry.path()?; + if is_dora_manifest_path(path.as_ref()) { + let mut manifest = String::new(); + entry.read_to_string(&mut manifest)?; + return PublishManifest::from_dora_toml_str(&manifest); + } + } + + bail!("archive does not contain `Dora.toml`") +} + +fn is_dora_manifest_path(path: &Path) -> bool { + path.file_name() + .and_then(|name| name.to_str()) + .map(|name| name == "Dora.toml") + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use std::path::PathBuf; + use tar::Header; + use tempfile::tempdir; + + #[test] + fn builds_record_and_validates_archive() { + let package_dir = create_test_package_dir().unwrap(); + + let (record, archive) = build_published_package(package_dir.path()).unwrap(); + + assert_eq!(record.name, "camera_node"); + assert_eq!(record.version.to_string(), "0.1.0"); + validate_published_archive(&record, &archive).unwrap(); + } + + #[test] + fn archive_contains_manifest() { + let package_dir = create_test_package_dir().unwrap(); + let archive = create_package_archive(package_dir.path()).unwrap(); + let manifest = read_manifest_from_archive(&archive).unwrap(); + + assert_eq!(manifest.name, "camera_node"); + } + + #[test] + fn validation_fails_for_checksum_mismatch() { + let package_dir = create_test_package_dir().unwrap(); + let (mut record, archive) = build_published_package(package_dir.path()).unwrap(); + record.checksum = "sha256:deadbeef".to_owned(); + + let err = validate_published_archive(&record, &archive) + .unwrap_err() + .to_string(); + + assert!(err.contains("checksum mismatch")); + } + + #[test] + fn validation_fails_without_manifest() { + let bytes = { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + { + let mut builder = Builder::new(&mut encoder); + let content = b"hello"; + let mut header = Header::new_gnu(); + header.set_size(content.len() as u64); + header.set_mode(0o644); + header.set_cksum(); + builder + .append_data( + &mut header, + PathBuf::from("README.md"), + Cursor::new(content), + ) + .unwrap(); + builder.finish().unwrap(); + } + encoder.finish().unwrap() + }; + + let err = read_manifest_from_archive(&bytes).unwrap_err().to_string(); + assert!(err.contains("does not contain `Dora.toml`")); + } + + fn create_test_package_dir() -> eyre::Result { + let dir = tempdir()?; + std::fs::create_dir_all(dir.path().join("camera_node"))?; + std::fs::write( + dir.path().join("Dora.toml"), + r#" +[package] +name = "camera_node" +version = "0.1.0" + +[dependencies] +yolo = { version = "^1.2.0" } +"#, + )?; + let mut source = std::fs::File::create(dir.path().join("camera_node").join("main.py"))?; + source.write_all(b"print('hello')\n")?; + Ok(dir) + } +} diff --git a/binaries/cli/src/publish_metadata.rs b/binaries/cli/src/publish_metadata.rs new file mode 100644 index 000000000..c1db2b546 --- /dev/null +++ b/binaries/cli/src/publish_metadata.rs @@ -0,0 +1,199 @@ +use eyre::{Context, bail}; +use semver::Version; +use std::{collections::BTreeMap, path::Path}; + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct PublishedDependency { + pub name: String, + pub requirement: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct PublishedPackageRecord { + pub name: String, + pub version: Version, + #[serde(default)] + pub dependencies: Vec, + pub checksum: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PublishManifest { + pub name: String, + pub version: Version, + pub dependencies: Vec, +} + +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct RawDoraToml { + package: RawPackageSection, + #[serde(default)] + dependencies: BTreeMap, +} + +#[derive(Debug, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct RawPackageSection { + name: String, + version: String, +} + +#[derive(Debug, serde::Deserialize)] +#[serde(untagged)] +enum RawDependencySpec { + Version(String), + Detailed { + version: Option, + path: Option, + git: Option, + rev: Option, + }, +} + +impl PublishManifest { + pub fn from_dora_toml_path(path: &Path) -> eyre::Result { + let raw = std::fs::read_to_string(path) + .with_context(|| format!("failed to read Dora metadata at `{}`", path.display()))?; + Self::from_dora_toml_str(&raw) + .with_context(|| format!("failed to parse Dora metadata at `{}`", path.display())) + } + + pub fn from_dora_toml_str(raw: &str) -> eyre::Result { + let manifest: RawDoraToml = toml::from_str(raw)?; + normalize_manifest(manifest) + } +} + +impl PublishedPackageRecord { + pub fn from_manifest(manifest: PublishManifest, checksum: String) -> Self { + Self { + name: manifest.name, + version: manifest.version, + dependencies: manifest.dependencies, + checksum, + } + } +} + +fn normalize_manifest(manifest: RawDoraToml) -> eyre::Result { + validate_package_name(&manifest.package.name)?; + let version = Version::parse(&manifest.package.version).with_context(|| { + format!( + "package version `{}` is not valid semver", + manifest.package.version + ) + })?; + + let mut dependencies = Vec::new(); + for (name, spec) in manifest.dependencies { + validate_package_name(&name)?; + let requirement = match spec { + RawDependencySpec::Version(requirement) => requirement, + RawDependencySpec::Detailed { + version: Some(requirement), + path: None, + git: None, + rev: None, + } => requirement, + RawDependencySpec::Detailed { path: Some(_), .. } => bail!( + "dependency `{name}` uses `path`, which is not publishable in registry metadata" + ), + RawDependencySpec::Detailed { git: Some(_), .. } => bail!( + "dependency `{name}` uses `git`, which is not publishable in registry metadata" + ), + RawDependencySpec::Detailed { rev: Some(_), .. } => { + bail!("dependency `{name}` sets `rev` without a publishable `version` source") + } + RawDependencySpec::Detailed { version: None, .. } => { + bail!("dependency `{name}` must declare a version requirement to be published") + } + }; + + semver::VersionReq::parse(&requirement).with_context(|| { + format!("dependency `{name}` has invalid version requirement `{requirement}`") + })?; + + dependencies.push(PublishedDependency { name, requirement }); + } + + Ok(PublishManifest { + name: manifest.package.name, + version, + dependencies, + }) +} + +fn validate_package_name(name: &str) -> eyre::Result<()> { + if name.is_empty() { + bail!("package/dependency name must not be empty"); + } + if !name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '-' || ch == '_') + { + bail!("invalid package/dependency name `{name}`"); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_publishable_manifest() { + let manifest = PublishManifest::from_dora_toml_str( + r#" +[package] +name = "camera_node" +version = "0.1.0" + +[dependencies] +yolo = { version = "^1.2.0" } +math = ">=0.3.0,<1.0.0" +"#, + ) + .unwrap(); + + assert_eq!(manifest.name, "camera_node"); + assert_eq!(manifest.version, Version::parse("0.1.0").unwrap()); + assert_eq!(manifest.dependencies.len(), 2); + } + + #[test] + fn rejects_path_dependency() { + let err = PublishManifest::from_dora_toml_str( + r#" +[package] +name = "camera_node" +version = "0.1.0" + +[dependencies] +vision = { path = "../vision-node" } +"#, + ) + .unwrap_err() + .to_string(); + + assert!(err.contains("not publishable")); + } + + #[test] + fn rejects_invalid_version_requirement() { + let err = PublishManifest::from_dora_toml_str( + r#" +[package] +name = "camera_node" +version = "0.1.0" + +[dependencies] +yolo = { version = "not-a-version" } +"#, + ) + .unwrap_err() + .to_string(); + + assert!(err.contains("invalid version requirement")); + } +}