wip

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Author: joseph-isaacs

pyproject.toml

@@ -36,6 +36,7 @@ dev = [
     "urllib3>=2.6.3",
     "filelock>=3.20.3",
     "protobuf>=6.33.5",
+    "jsonschema>=4.0",
 ]
 
 [tool.uv]

uv.lock

@@ -201,13 +201,13 @@ Uses the Epoch C API (`session.write_options()` /
 1. Detect version from nearest git tag at HEAD (or `<REF>`)
 2. Generate fixtures (from current tree, or from a worktree at `<REF>`)
 3. Fetch previous version's manifest, merge `since` values, enforce additive-only
-4. Upload `.vortex` files + `manifest.json` to `v{version}/`
+4. Upload `.vortex` files + `manifest.json` to `v{version}/arrays/`
 5. Update `versions.json`
 
 #### `check [--versions <CSV>] [--store <SPEC>] [--exclude <CSV>]`
 
 1. Read `versions.json` from store
-2. For each version, download `manifest.json` + all `.vortex` files
+2. For each version, download `arrays/manifest.json` + all `.vortex` files
 3. Run `vortex-compat check --dir <tmpdir> --mode subset`
 4. Aggregate results, exit 1 if any failures
 
@@ -266,16 +266,19 @@ A JSON array of version strings:
 store/
 ├── versions.json
 ├── v0.62.0/
-│   ├── manifest.json
-│   ├── primitives.vortex
-│   └── ...
+│   └── arrays/
+│       ├── manifest.json
+│       ├── primitives.vortex
+│       └── ...
 └── v0.63.0/
-    ├── manifest.json
-    └── ...
+    └── arrays/
+        ├── manifest.json
+        └── ...
 ```
 
-Each version gets a directory named `v{version}`. Re-publishing overwrites
-the existing directory.
+Each version gets a directory named `v{version}` with an `arrays/`
+subdirectory for fixture files. Re-publishing overwrites the existing
+directory.
 
 ### Per-version manifest
 

vortex-test/compat-gen/DESIGN.md

@@ -13,13 +13,13 @@
 
 Quick start:
     # Generate + publish for HEAD (version auto-detected from latest tag)
-    python compat.py publish
+    uv run compat.py publish
 
     # Publish from an older tag
-    python compat.py publish --git-ref v0.62.0
+    uv run compat.py publish --git-ref v0.62.0
 
     # Check all published versions against current code
-    python compat.py check
+    uv run compat.py check
 """
 
 from __future__ import annotations
@@ -37,6 +37,8 @@
 from urllib.error import HTTPError
 from urllib.request import urlopen
 
+import jsonschema
+
 DEFAULT_STORE = "s3://vortex-compat-fixtures"
 CARGO_BIN = "vortex-compat"
 
@@ -60,26 +62,26 @@
 
 examples:
   # Publish from HEAD (version from latest tag)
-  python compat.py publish
-  python compat.py publish --dry-run
+  uv run compat.py publish
+  uv run compat.py publish --dry-run
 
   # Publish from an older tag via worktree
-  python compat.py publish --git-ref v0.62.0
+  uv run compat.py publish --git-ref v0.62.0
 
   # Generate locally without publishing
-  python compat.py generate --output /tmp/fixtures
-  python compat.py generate --output /tmp/fixtures --git-ref v0.62.0
+  uv run compat.py generate --output /tmp/fixtures
+  uv run compat.py generate --output /tmp/fixtures --git-ref v0.62.0
 
   # Check all versions, or specific ones
-  python compat.py check
-  python compat.py check --versions 0.62.0,0.63.0
+  uv run compat.py check
+  uv run compat.py check --versions 0.62.0,0.63.0
 
   # Inspect store contents
-  python compat.py list
-  python compat.py list --version 0.62.0
+  uv run compat.py list
+  uv run compat.py list --version 0.62.0
 
   # Validate additive-only manifest property
-  python compat.py validate-manifest
+  uv run compat.py validate-manifest
 """
 
 
@@ -139,7 +141,7 @@ def list_versions(self) -> list[str]:
         versions = []
         for entry in self.root.iterdir():
             if entry.is_dir() and entry.name.startswith("v"):
-                manifest = entry / "manifest.json"
+                manifest = entry / "arrays" / "manifest.json"
                 if manifest.exists():
                     versions.append(entry.name[1:])  # strip 'v' prefix
         versions.sort(key=_version_sort_key)
@@ -233,11 +235,43 @@ def _version_from_ref(git_ref: str | None = None) -> str:
 # ---------------------------------------------------------------------------
 
 
+MANIFEST_SCHEMA = {
+    "type": "object",
+    "required": ["version", "generated_at", "fixtures"],
+    "properties": {
+        "version": {"type": "string"},
+        "generated_at": {"type": "string"},
+        "fixtures": {
+            "type": "array",
+            "items": {
+                "type": "object",
+                "required": ["name", "since"],
+                "properties": {
+                    "name": {"type": "string"},
+                    "description": {"type": "string"},
+                    "since": {"type": "string"},
+                },
+            },
+        },
+    },
+}
+
+
+def _validate_manifest(manifest: dict, version: str) -> None:
+    """Validate manifest against the JSON schema."""
+    try:
+        jsonschema.validate(manifest, MANIFEST_SCHEMA)
+    except jsonschema.ValidationError as e:
+        raise ValueError(f"v{version} manifest: {e.message} (at path: {'/'.join(str(p) for p in e.absolute_path)})") from e
+
+
 def _read_manifest(store: Store, version: str) -> dict | None:
-    data = store.read(f"v{version}/manifest.json")
+    data = store.read(f"v{version}/arrays/manifest.json")
     if data is None:
         return None
-    return json.loads(data)
+    manifest = json.loads(data)
+    _validate_manifest(manifest, version)
+    return manifest
 
 
 def _merge_manifest(
@@ -255,10 +289,7 @@ def _merge_manifest(
     for f in fixtures_json["fixtures"]:
         name = f["name"]
         since = prev_since.get(name, version)
-        entry = {"name": name, "description": f["description"], "since": since}
-        if "expected_encodings" in f:
-            entry["expected_encodings"] = f["expected_encodings"]
-        entries.append(entry)
+        entries.append({"name": name, "description": f["description"], "since": since})
 
     # Additive-only enforcement.
     current_names = {e["name"] for e in entries}
@@ -347,10 +378,7 @@ def cmd_generate(args: argparse.Namespace) -> None:
     fixtures_json = json.loads((output / "fixtures.json").read_text())
     entries = []
     for f in fixtures_json["fixtures"]:
-        entry = {"name": f["name"], "description": f["description"], "since": version}
-        if "expected_encodings" in f:
-            entry["expected_encodings"] = f["expected_encodings"]
-        entries.append(entry)
+        entries.append({"name": f["name"], "description": f["description"], "since": version})
     manifest = {
         "version": version,
         "generated_at": datetime.now(timezone.utc).isoformat(),
@@ -397,11 +425,11 @@ def cmd_publish(args: argparse.Namespace) -> None:
         for entry in manifest["fixtures"]:
             name = entry["name"]
             local = output / name
-            key = f"v{version}/{name}"
+            key = f"v{version}/arrays/{name}"
             store.write_file(key, local)
             _info(f"  uploaded {name}")
 
-        store.write(f"v{version}/manifest.json", manifest_json.encode())
+        store.write(f"v{version}/arrays/manifest.json", manifest_json.encode())
         _info("  uploaded manifest.json")
 
         if version not in versions:
@@ -448,7 +476,7 @@ def cmd_check(args: argparse.Namespace) -> None:
 
             for entry in manifest["fixtures"]:
                 name = entry["name"]
-                data = store.read(f"v{version}/{name}")
+                data = store.read(f"v{version}/arrays/{name}")
                 if data is None:
                     _info(f"  v{version}: {name} not found in store")
                     continue
@@ -652,8 +680,8 @@ def main() -> None:
         ),
         epilog=(
             "examples:\n"
-            "  python compat.py generate --output ./out\n"
-            "  python compat.py generate --output ./out --git-ref v0.62.0"
+            "  uv run compat.py generate --output ./out\n"
+            "  uv run compat.py generate --output ./out --git-ref v0.62.0"
         ),
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -679,10 +707,10 @@ def main() -> None:
         ),
         epilog=(
             "examples:\n"
-            "  python compat.py publish\n"
-            "  python compat.py publish --dry-run\n"
-            "  python compat.py publish --git-ref v0.62.0\n"
-            "  python compat.py publish --store /tmp/store"
+            "  uv run compat.py publish\n"
+            "  uv run compat.py publish --dry-run\n"
+            "  uv run compat.py publish --git-ref v0.62.0\n"
+            "  uv run compat.py publish --store /tmp/store"
         ),
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -715,9 +743,9 @@ def main() -> None:
         ),
         epilog=(
             "examples:\n"
-            "  python compat.py check\n"
-            "  python compat.py check --versions 0.62.0,0.63.0\n"
-            "  python compat.py check --store /tmp/store"
+            "  uv run compat.py check\n"
+            "  uv run compat.py check --versions 0.62.0,0.63.0\n"
+            "  uv run compat.py check --store /tmp/store"
         ),
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -740,9 +768,9 @@ def main() -> None:
         description="Inspect the contents of a fixture store.",
         epilog=(
             "examples:\n"
-            "  python compat.py list\n"
-            "  python compat.py list --version 0.62.0\n"
-            "  python compat.py list --store /tmp/store"
+            "  uv run compat.py list\n"
+            "  uv run compat.py list --version 0.62.0\n"
+            "  uv run compat.py list --store /tmp/store"
         ),
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )
@@ -761,8 +789,8 @@ def main() -> None:
         ),
         epilog=(
             "examples:\n"
-            "  python compat.py validate-manifest\n"
-            "  python compat.py validate-manifest --store /tmp/store"
+            "  uv run compat.py validate-manifest\n"
+            "  uv run compat.py validate-manifest --store /tmp/store"
         ),
         formatter_class=argparse.RawDescriptionHelpFormatter,
     )

vortex-test/compat-gen/scripts/compat.py

@@ -1,11 +1,6 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-// Epoch C adapter — for Vortex v0.58.0 through HEAD
-//
-// Write: session.write_options(), returns WriteSummary, takes &mut sink
-// Read:  session.open_options().open_buffer(buf) (sync), into_array_stream() (async)
-
 use std::path::Path;
 use std::sync::Arc;
 
@@ -22,10 +17,11 @@ use vortex_array::stream::ArrayStreamAdapter;
 use vortex_array::stream::ArrayStreamExt;
 use vortex_buffer::ByteBuffer;
 use vortex_error::VortexResult;
+use vortex_error::vortex_err;
 use vortex_session::VortexSession;
 
 fn runtime() -> VortexResult<Runtime> {
-    Runtime::new().map_err(|e| vortex_error::vortex_err!("failed to create tokio runtime: {e}"))
+    Runtime::new().map_err(|e| vortex_err!("failed to create tokio runtime: {e}"))
 }
 
 /// Write a sequence of array chunks as a `.vortex` file with no compression.
@@ -41,12 +37,8 @@ pub fn write_file(path: &Path, chunk: ArrayRef) -> VortexResult<()> {
         let session = VortexSession::default().with_tokio();
         let mut file = tokio::fs::File::create(path)
             .await
-            .map_err(|e| vortex_error::vortex_err!("failed to create {}: {e}", path.display()))?;
-        let _summary = session
-            .write_options()
-            .with_strategy(strategy)
-            .write(&mut file, stream)
-            .await?;
+            .map_err(|e| vortex_err!("failed to create {}: {e}", path.display()))?;
+        let _summary = session.write_options().write(&mut file, stream).await?;
         Ok(())
     })
 }

vortex-test/compat-gen/src/adapter.rs

@@ -3,6 +3,7 @@
 
 use std::path::Path;
 
+use clap::ValueEnum;
 use serde::Serialize;
 use vortex_array::IntoArray;
 use vortex_array::arrays::ChunkedArray;
@@ -16,6 +17,7 @@ use crate::adapter;
 use crate::fixtures::all_fixtures;
 
 /// How to handle mismatches between directory and known fixtures.
+#[derive(Clone, ValueEnum)]
 pub enum Mode {
     /// Directory must match fixtures exactly.
     Exact,

vortex-test/compat-gen/src/check.rs

@@ -5,19 +5,16 @@ use std::path::Path;
 
 use arrow_array::RecordBatch;
 use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
-use vortex::layout::LayoutId;
 use vortex_array::ArrayRef;
 use vortex_array::IntoArray;
 use vortex_array::arrays::ChunkedArray;
 use vortex_array::arrays::Primitive;
 use vortex_array::arrays::Struct;
 use vortex_array::arrays::VarBin;
 use vortex_array::arrow::FromArrowArray;
-use vortex_array::vtable::ArrayId;
 use vortex_error::VortexResult;
 use vortex_error::vortex_err;
 
-use super::ExpectedEncoding;
 use super::Fixture;
 
 /// First partition of ClickBench hits, limited to 1000 rows.
@@ -37,16 +34,6 @@ impl ArrayFixture for ClickBenchHits1kFixture {
         "First 1000 rows of ClickBench hits_0 partition (wide real-world schema)"
     }
 
-    fn expected_encodings(&self) -> Vec<ExpectedEncoding> {
-        vec![
-            ExpectedEncoding::Array(ArrayId::new_ref("vortex.primitive")),
-            ExpectedEncoding::Array(ArrayId::new_ref("vortex.varbin")),
-            ExpectedEncoding::Array(ArrayId::new_ref("vortex.struct")),
-            ExpectedEncoding::Layout(LayoutId::new_ref("vortex.flat")),
-            ExpectedEncoding::Layout(LayoutId::new_ref("vortex.struct")),
-        ]
-    }
-
     fn setup(&self, tmp_dir: &Path) -> VortexResult<()> {
         let parquet_path = tmp_dir.join(PARQUET_FILENAME);
         if parquet_path.exists() {