py cli

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Author: joseph-isaacs

Cargo.lock

@@ -216,6 +216,7 @@ rustc-hash = "2.1"
 serde = "1.0.220"
 serde_json = "1.0.138"
 serde_test = "1.0.176"
+sha2 = "0.10"
 simdutf8 = "0.1.5"
 similar = "2.7.0"
 sketches-ddsketch = "0.3.0"

Cargo.toml

@@ -47,3 +47,4 @@ reqwest = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 serde = { workspace = true, features = ["derive"] }
 serde_json = { workspace = true }
+sha2 = { workspace = true }

vortex-test/compat-gen/Cargo.toml

@@ -253,11 +253,10 @@ def _version_from_ref(git_ref: str | None = None) -> str:
             "type": "array",
             "items": {
                 "type": "object",
-                "required": ["name", "since"],
+                "required": ["name"],
                 "properties": {
                     "name": {"type": "string"},
                     "description": {"type": "string"},
-                    "since": {"type": "string"},
                     "sha256": {"type": "string"},
                 },
             },
@@ -288,7 +287,7 @@ def _read_manifest(store: Store, version: str) -> dict | None:
     # Upgrade legacy format: flat list of filenames -> new object format.
     if isinstance(manifest.get("fixtures"), list) and manifest["fixtures"] and isinstance(manifest["fixtures"][0], str):
         _info(f"  upgrading legacy manifest format for v{version}")
-        manifest["fixtures"] = [{"name": n, "description": "", "since": version} for n in manifest["fixtures"]]
+        manifest["fixtures"] = [{"name": n, "description": "", "sha256": ""} for n in manifest["fixtures"]]
 
     _validate_manifest(manifest, version)
     # Stash the prefix so callers know where to fetch fixture files.
@@ -298,26 +297,22 @@ def _read_manifest(store: Store, version: str) -> dict | None:
 
 def _merge_manifest(
     store: Store, fixtures_json: dict, version: str, prev_version: str | None,
-    output: Path,
 ) -> dict:
-    """Build a manifest for `version`, carrying forward `since` from prev_version."""
+    """Build a manifest for `version`, using sha256 from Rust-generated fixtures.json."""
     entries = []
-    prev_since: dict[str, str] = {}
+    prev_names: set[str] = set()
 
     if prev_version:
         prev_manifest = _read_manifest(store, prev_version)
         if prev_manifest:
-            prev_since = {e["name"]: e["since"] for e in prev_manifest["fixtures"]}
+            prev_names = {e["name"] for e in prev_manifest["fixtures"]}
 
     for f in fixtures_json["fixtures"]:
-        name = f["name"]
-        since = prev_since.get(name, version)
-        sha = hashlib.sha256((output / name).read_bytes()).hexdigest()
-        entries.append({"name": name, "description": f["description"], "since": since, "sha256": sha})
+        entries.append({"name": f["name"], "description": f["description"], "sha256": f["sha256"]})
 
     # Additive-only enforcement.
     current_names = {e["name"] for e in entries}
-    missing = [n for n in prev_since if n not in current_names]
+    missing = [n for n in prev_names if n not in current_names]
     if missing:
         print(
             f"error: fixtures removed since v{prev_version}: {', '.join(missing)}",
@@ -345,12 +340,11 @@ def cmd_generate(args: argparse.Namespace) -> None:
 
     _run_rust_generate(output)
 
-    # Read fixtures.json and write a versioned manifest.
+    # Read fixtures.json (with sha256 from Rust) and write a versioned manifest.
     fixtures_json = json.loads((output / "fixtures.json").read_text())
     entries = []
     for f in fixtures_json["fixtures"]:
-        sha = hashlib.sha256((output / f["name"]).read_bytes()).hexdigest()
-        entries.append({"name": f["name"], "description": f["description"], "since": version, "sha256": sha})
+        entries.append({"name": f["name"], "description": f["description"], "sha256": f["sha256"]})
     manifest = {
         "version": version,
         "generated_at": datetime.now(timezone.utc).isoformat(),
@@ -398,7 +392,7 @@ def _publish_full(
         if prev:
             _info(f"previous version: {prev}")
 
-        manifest = _merge_manifest(store, fixtures_json, version, prev, output)
+        manifest = _merge_manifest(store, fixtures_json, version, prev)
         manifest_json = json.dumps(manifest, indent=2) + "\n"
 
         if args.dry_run:
@@ -541,8 +535,7 @@ def _publish_update(
         new_entries = existing_manifest["fixtures"][:]
         for f in fixtures_json["fixtures"]:
             if f["name"] in new_fixture_names:
-                sha = hashlib.sha256((output / f["name"]).read_bytes()).hexdigest()
-                new_entries.append({"name": f["name"], "description": f["description"], "since": version, "sha256": sha})
+                new_entries.append({"name": f["name"], "description": f["description"], "sha256": f["sha256"]})
 
         updated_manifest = {
             "version": version,

vortex-test/compat-gen/scripts/compat.py

@@ -5,6 +5,8 @@ use std::path::Path;
 use std::thread;
 
 use serde::Serialize;
+use sha2::Digest;
+use sha2::Sha256;
 use vortex_array::ArrayRef;
 use vortex_error::VortexResult;
 use vortex_error::vortex_bail;
@@ -24,6 +26,7 @@ struct FixturesJson {
 struct FixtureInfo {
     name: String,
     description: String,
+    sha256: String,
 }
 
 /// Generate all fixtures into `output_dir`.
@@ -68,9 +71,13 @@ pub fn generate(output_dir: &Path, exclude: &[String]) -> VortexResult<()> {
         check_expected_encodings(array, *fixture)?;
         let path = output_dir.join(fixture.name());
         adapter::write_file(&path, array.clone())?;
+        let file_bytes = std::fs::read(&path)
+            .map_err(|e| vortex_err!("failed to read back {}: {e}", path.display()))?;
+        let sha256 = format!("{:x}", Sha256::digest(&file_bytes));
         infos.push(FixtureInfo {
             name: fixture.name().to_string(),
             description: fixture.description().to_string(),
+            sha256,
         });
         eprintln!("  wrote {}", fixture.name());
     }