wip

Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>

Author: joseph-isaacs

vortex-test/compat-gen/scripts/compat.py

@@ -8,8 +8,7 @@
 Manages fixture versions in S3 (or local directories) by calling the thin
 `vortex-compat` Rust binary for generation and checking.  The Rust binary
 handles only two things: generating .vortex files and comparing them.
-Everything else (versioning, S3 upload/download, manifest merging, worktree
-management) lives here.
+Everything else (versioning, S3 upload/download, manifest merging) lives here.
 
 Quick start:
     # Generate + publish for HEAD (version auto-detected from latest tag)
@@ -65,7 +64,7 @@
   uv run compat.py publish
   uv run compat.py publish --dry-run
 
-  # Publish from an older tag via worktree
+  # Publish using an older tag for version detection
   uv run compat.py publish --git-ref v0.62.0
 
   # Generate locally without publishing
@@ -309,54 +308,6 @@ def _merge_manifest(
     }
 
 
-# ---------------------------------------------------------------------------
-# Worktree helpers
-# ---------------------------------------------------------------------------
-
-
-def _worktree_generate(git_ref: str, output_dir: Path) -> None:
-    """Create a git worktree at `git_ref`, build vortex-compat, and generate fixtures."""
-    repo_root = Path(
-        subprocess.run(
-            ["git", "rev-parse", "--show-toplevel"],
-            capture_output=True,
-            text=True,
-            check=True,
-        ).stdout.strip()
-    )
-
-    worktree_dir = Path(tempfile.mkdtemp(prefix=f"vortex-compat-wt-{git_ref}-"))
-    try:
-        _info(f"creating worktree at {git_ref} in {worktree_dir}")
-        _run_cmd(
-            ["git", "-C", str(repo_root), "worktree", "add", str(worktree_dir), git_ref],
-            check=True,
-        )
-
-        _info(f"building vortex-compat at {git_ref}...")
-        _run_cmd(
-            ["cargo", "build", "-p", CARGO_BIN, "--release"],
-            check=True,
-            cwd=worktree_dir,
-        )
-
-        bin_path = worktree_dir / "target" / "release" / CARGO_BIN
-        if not bin_path.exists():
-            print(f"error: binary not found at {bin_path}", file=sys.stderr)
-            sys.exit(1)
-
-        _info(f"generating fixtures with {git_ref} binary...")
-        _run_cmd([str(bin_path), "generate", "--output", str(output_dir)], check=True)
-
-    finally:
-        _run_cmd(
-            ["git", "-C", str(repo_root), "worktree", "remove", "--force", str(worktree_dir)],
-            check=False,
-        )
-        if worktree_dir.exists():
-            shutil.rmtree(worktree_dir, ignore_errors=True)
-
-
 # ---------------------------------------------------------------------------
 # Commands
 # ---------------------------------------------------------------------------
@@ -365,14 +316,10 @@ def _worktree_generate(git_ref: str, output_dir: Path) -> None:
 def cmd_generate(args: argparse.Namespace) -> None:
     """Generate fixtures locally, then write a proper manifest."""
     output = Path(args.output)
-    git_ref = args.git_ref
     exclude = _parse_exclude(args)
-    version = _version_from_ref(git_ref)
+    version = _version_from_ref(args.git_ref)
 
-    if git_ref:
-        _worktree_generate(git_ref, output)
-    else:
-        _run_rust_generate(output, exclude=exclude)
+    _run_rust_generate(output, exclude=exclude)
 
     # Read fixtures.json and write a versioned manifest.
     fixtures_json = json.loads((output / "fixtures.json").read_text())
@@ -399,10 +346,7 @@ def cmd_publish(args: argparse.Namespace) -> None:
         output = Path(tmpdir) / "fixtures"
 
         _info("generating fixtures...")
-        if git_ref:
-            _worktree_generate(git_ref, output)
-        else:
-            _run_rust_generate(output, exclude=exclude)
+        _run_rust_generate(output, exclude=exclude)
 
         fixtures_json = json.loads((output / "fixtures.json").read_text())
 
@@ -674,9 +618,9 @@ def main() -> None:
         "generate",
         help="Generate fixtures locally",
         description=(
-            "Build all fixture .vortex files and write them to a directory.\n"
-            "Version is auto-detected from the nearest git tag at HEAD\n"
-            "(or at --git-ref if specified)."
+            "Build all fixture .vortex files using the current binary and write\n"
+            "them to a directory. Version is auto-detected from the nearest git\n"
+            "tag at HEAD (or at --git-ref if specified)."
         ),
         epilog=(
             "examples:\n"
@@ -688,8 +632,9 @@ def main() -> None:
     p.add_argument("--output", required=True, help="Output directory")
     p.add_argument(
         "--git-ref",
-        help="Git ref (tag/branch/SHA) to build from via worktree. "
-        "Version is derived from the nearest tag at this ref.",
+        help="Git ref for version detection (e.g. v0.62.0). "
+        "Version is derived from the nearest tag at this ref. "
+        "Fixtures are always built with the current binary.",
     )
     p.add_argument(
         "--exclude",
@@ -719,8 +664,9 @@ def main() -> None:
     )
     p.add_argument(
         "--git-ref",
-        help="Git ref to build from via worktree (e.g. v0.62.0). "
-        "Version is derived from the nearest tag at this ref.",
+        help="Git ref for version detection (e.g. v0.62.0). "
+        "Version is derived from the nearest tag at this ref. "
+        "Fixtures are always built with the current binary.",
     )
     p.add_argument(
         "--dry-run",

vortex-test/compat-gen/src/adapter.rs

@@ -12,7 +12,6 @@ use vortex::file::WriteOptionsSessionExt;
 use vortex::io::session::RuntimeSessionExt;
 use vortex::layout::layouts::flat::writer::FlatLayoutStrategy;
 use vortex_array::ArrayRef;
-use vortex_array::DynArray;
 use vortex_array::stream::ArrayStreamAdapter;
 use vortex_array::stream::ArrayStreamExt;
 use vortex_buffer::ByteBuffer;
@@ -38,7 +37,11 @@ pub fn write_file(path: &Path, chunk: ArrayRef) -> VortexResult<()> {
         let mut file = tokio::fs::File::create(path)
             .await
             .map_err(|e| vortex_err!("failed to create {}: {e}", path.display()))?;
-        let _summary = session.write_options().write(&mut file, stream).await?;
+        let _summary = session
+            .write_options()
+            .with_strategy(strategy)
+            .write(&mut file, stream)
+            .await?;
         Ok(())
     })
 }

vortex-test/compat-gen/src/check.rs

@@ -5,8 +5,6 @@ use std::path::Path;
 
 use clap::ValueEnum;
 use serde::Serialize;
-use vortex_array::IntoArray;
-use vortex_array::arrays::ChunkedArray;
 use vortex_array::assert_arrays_eq;
 use vortex_buffer::ByteBuffer;
 use vortex_error::VortexResult;
@@ -125,7 +123,7 @@ pub fn check(dir: &Path, mode: Mode, exclude: &[String]) -> VortexResult<()> {
             continue;
         }
         let expected = match fixture.build(&tmp_dir) {
-            Ok(chunks) => chunks,
+            Ok(arr) => arr,
             Err(e) => {
                 result.failed.push(FailedFixture {
                     name: fixture.name().to_string(),
@@ -158,19 +156,9 @@ pub fn check(dir: &Path, mode: Mode, exclude: &[String]) -> VortexResult<()> {
         };
 
         // Compare.
-        match compare_arrays(&actual, &expected) {
-            Ok(()) => {
-                eprintln!("  pass {}", fixture.name());
-                result.passed.push(fixture.name().to_string());
-            }
-            Err(e) => {
-                eprintln!("  FAIL {}", fixture.name());
-                result.failed.push(FailedFixture {
-                    name: fixture.name().to_string(),
-                    error: e.to_string(),
-                });
-            }
-        }
+        assert_arrays_eq!(actual, expected);
+        eprintln!("  pass {}", fixture.name());
+        result.passed.push(fixture.name().to_string());
     }
 
     // Print JSON result to stdout.
@@ -195,15 +183,3 @@ pub fn check(dir: &Path, mode: Mode, exclude: &[String]) -> VortexResult<()> {
 
     Ok(())
 }
-
-fn compare_arrays(
-    actual: &[vortex_array::ArrayRef],
-    expected: &[vortex_array::ArrayRef],
-) -> VortexResult<()> {
-    let actual_dtype = actual[0].dtype().clone();
-    let expected_dtype = expected[0].dtype().clone();
-    let actual_arr = ChunkedArray::try_new(actual.to_vec(), actual_dtype)?.into_array();
-    let expected_arr = ChunkedArray::try_new(expected.to_vec(), expected_dtype)?.into_array();
-    assert_arrays_eq!(actual_arr, expected_arr);
-    Ok(())
-}

vortex-test/compat-gen/src/fixtures/clickbench.rs

@@ -12,26 +12,27 @@ use vortex_array::arrays::Primitive;
 use vortex_array::arrays::Struct;
 use vortex_array::arrays::VarBin;
 use vortex_array::arrow::FromArrowArray;
+use vortex_array::vtable::ArrayId;
 use vortex_error::VortexResult;
 use vortex_error::vortex_err;
 
-use super::Fixture;
+use super::ArrayFixture;
 
 /// First partition of ClickBench hits, limited to 1000 rows.
 const CLICKBENCH_URL: &str =
     "https://pub-3ba949c0f0354ac18db1f0f14f0a2c52.r2.dev/clickbench/parquet_many/hits_0.parquet";
 
 const PARQUET_FILENAME: &str = "clickbench_hits_0.parquet";
 
-pub struct ClickBenchHits1kFixture;
+struct ClickBenchHits1kFixture;
 
 impl ArrayFixture for ClickBenchHits1kFixture {
     fn name(&self) -> &str {
         "clickbench_hits_1k.vortex"
     }
 
     fn description(&self) -> &str {
-        "First 1000 rows of ClickBench hits_0 partition (wide real-world schema)"
+        "First 1000 rows of ClickBench hits dataset with wide schema of primitives and strings"
     }
 
     fn setup(&self, tmp_dir: &Path) -> VortexResult<()> {
@@ -49,7 +50,7 @@ impl ArrayFixture for ClickBenchHits1kFixture {
         Ok(())
     }
 
-    fn build(&self, tmp_dir: &Path) -> VortexResult<Vec<ArrayRef>> {
+    fn build(&self, tmp_dir: &Path) -> VortexResult<ArrayRef> {
         let parquet_path = tmp_dir.join(PARQUET_FILENAME);
         let file_bytes = std::fs::read(&parquet_path)
             .map_err(|e| vortex_err!("failed to read cached parquet: {e}"))?;

vortex-test/compat-gen/src/fixtures/mod.rs

@@ -8,17 +8,19 @@ mod tpch;
 use std::path::Path;
 
 use vortex_array::ArrayRef;
+use vortex_array::ArrayVisitorExt;
+use vortex_array::vtable::ArrayId;
 use vortex_error::VortexResult;
 use vortex_error::vortex_bail;
 
 /// A deterministic fixture that produces the same arrays every time.
 ///
 /// Lifecycle:
-/// 1. **Setup** (optional, async I/O) — download external data or prepare
+/// 1. **Setup** (optional, I/O) — download external data or prepare
 ///    intermediate files in `tmp_dir`. Run concurrently across fixtures.
 /// 2. **Build** (CPU, parallel) — construct arrays from scratch or from
 ///    data prepared during setup.
-pub trait Fixture: Send + Sync {
+pub trait ArrayFixture: Send + Sync {
     /// Unique name for this fixture, used as the output filename.
     fn name(&self) -> &str;
 
@@ -27,17 +29,57 @@ pub trait Fixture: Send + Sync {
 
     /// Optional setup phase for downloading external data or preparing files.
     ///
-    /// Called before `build()`. Runs concurrently across fixtures via
-    /// `tokio::spawn_blocking`. The default implementation is a no-op.
+    /// Called before `build()`. Runs concurrently across fixtures.
+    /// The default implementation is a no-op.
     fn setup(&self, _tmp_dir: &Path) -> VortexResult<()> {
         Ok(())
     }
 
-    /// Build the expected arrays, using `tmp_dir` for any data prepared
+    /// Build the expected array, using `tmp_dir` for any data prepared
     /// during `setup()`.
     ///
-    /// Must be deterministic. Returns a `Vec` to support chunked fixtures.
-    fn build(&self, tmp_dir: &Path) -> VortexResult<Vec<ArrayRef>>;
+    /// Must be deterministic under all versions of vortex.
+    fn build(&self, tmp_dir: &Path) -> VortexResult<ArrayRef>;
+
+    /// Encoding IDs that must appear somewhere in the array tree produced by [`Self::build`].
+    ///
+    /// Only include encodings that the fixture is specifically testing, not incidental ones
+    /// (e.g. a primitives fixture should not list struct even if it wraps values in one).
+    ///
+    /// An empty slice (the default) disables the check.
+    fn expected_encodings(&self) -> Vec<ArrayId> {
+        vec![]
+    }
+}
+
+/// Walk the array tree, collect encoding IDs, and assert that all expected encodings
+/// are present. This is a no-op when [`ArrayFixture::expected_encodings`] returns an empty vec.
+pub fn check_expected_encodings(array: &ArrayRef, fixture: &dyn ArrayFixture) -> VortexResult<()> {
+    let expected = fixture.expected_encodings();
+    if expected.is_empty() {
+        return Ok(());
+    }
+
+    let mut found: Vec<ArrayId> = Vec::new();
+    for node in array.depth_first_traversal() {
+        let id = node.encoding_id();
+        if !found.contains(&id) {
+            found.push(id);
+        }
+    }
+
+    let missing: Vec<&ArrayId> = expected.iter().filter(|id| !found.contains(id)).collect();
+
+    if !missing.is_empty() {
+        vortex_bail!(
+            "fixture '{}' is missing expected encodings: {:?} (found: {:?})",
+            fixture.name(),
+            missing,
+            found,
+        );
+    }
+
+    Ok(())
 }
 
 /// All registered fixtures.