difftest: nextest support and speedups

Cargo.toml

@@ -31,6 +31,7 @@ members = [
     "tests/compiletests",
     "tests/compiletests/deps-helper",
     "tests/difftests/bin",
+    "tests/difftests/runner",
     "tests/difftests/lib",
 ]
 
@@ -84,7 +85,7 @@ expect-test = "1.5.1"
 regex-lite = "0.1.9"
 
 # difftest libraries mirrored from difftest workspace
-difftest = { path = "tests/difftests/lib" }
+difftest-runner = { path = "tests/difftests/runner" }
 
 # External dependencies that need to be mentioned more than once.
 tracing = "0.1"

tests/difftests/bin/Cargo.toml

@@ -1,35 +1,26 @@
 [package]
 name = "difftests"
 version = "0.0.0"
-publish = false
 authors.workspace = true
 edition.workspace = true
 license.workspace = true
 repository.workspace = true
+publish = false
 
 # See rustc_codegen_spirv/Cargo.toml for details on these features
 [features]
 default = ["use-compiled-tools"]
-use-installed-tools = []
-use-compiled-tools = []
+use-installed-tools = ["difftest-runner/use-installed-tools"]
+use-compiled-tools = ["difftest-runner/use-compiled-tools"]
 
 [dependencies]
 anyhow = "1.0"
-tracing = "0.1"
-tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"] }
-tempfile = "3.5"
-tester = "0.9.1"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
-thiserror = "2.0.12"
-toml = { version = "0.9.2", default-features = false, features = ["parse", "serde"] }
-bytesize = "2.0.1"
-bytemuck = "1.21.0"
-difftest = { path = "../lib" }
-tabled = { version = "0.20.0", default-features = false, features = ["std"] }
+difftest-runner.workspace = true
 
 [lints]
 workspace = true
 
-[package.metadata.release]
-release = false
+[[test]]
+name = "difftests"
+path = "src/test.rs"
+harness = false

tests/difftests/bin/src/test.rs

@@ -0,0 +1,6 @@
+use anyhow::Result;
+use std::process::ExitCode;
+
+pub fn main() -> Result<ExitCode> {
+    difftest_runner::run()
+}

tests/difftests/runner/Cargo.toml

@@ -0,0 +1,37 @@
+[package]
+name = "difftest-runner"
+version = "0.0.0"
+publish = false
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+# See rustc_codegen_spirv/Cargo.toml for details on these features
+[features]
+default = ["use-compiled-tools"]
+use-installed-tools = []
+use-compiled-tools = []
+
+[dependencies]
+anyhow = "1.0"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3.20", features = ["fmt", "env-filter"] }
+libtest-mimic = "0.8.1"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+thiserror = "2.0.12"
+toml = { version = "0.9.2", default-features = false, features = ["parse", "serde"] }
+bytesize = "2.0.1"
+bytemuck = "1.21.0"
+difftest = { path = "../lib" }
+tabled = { version = "0.20.0", default-features = false, features = ["std"] }
+
+[dev-dependencies]
+tempfile = "3.5"
+
+[lints]
+workspace = true
+
+[package.metadata.release]
+release = false

tests/difftests/runner/src/lib.rs

@@ -0,0 +1,71 @@
+#![allow(clippy::exit)]
+
+use crate::testcase::collect_test_dirs;
+use anyhow::Result;
+use libtest_mimic::{Arguments, Trial};
+use runner::Runner;
+use std::process::ExitCode;
+use std::sync::Arc;
+use std::{
+    env, fs,
+    process::{self},
+};
+use tracing_subscriber::FmtSubscriber;
+
+mod differ;
+mod runner;
+mod testcase;
+
+pub fn run() -> Result<ExitCode> {
+    let subscriber = FmtSubscriber::builder()
+        .with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
+        .finish();
+    tracing::subscriber::set_global_default(subscriber).expect("Failed to set global subscriber");
+
+    let mut args = Arguments::from_args();
+
+    // If filters are provided that look like paths (contain '/'), convert them to test names
+    if let Some(filter) = &mut args.filter {
+        *filter = filter.replace('/', "::");
+    }
+
+    let tests = collect_tests()?;
+    Ok(libtest_mimic::run(&args, tests).exit_code())
+}
+
+fn collect_tests() -> Result<Vec<Trial>> {
+    // Find the manifest directory at compile time and locate tests in ../tests.
+    let manifest_dir = env!("CARGO_MANIFEST_DIR");
+    let base = std::path::Path::new(manifest_dir)
+        .join("../tests")
+        .canonicalize()
+        .expect("Failed to canonicalize tests directory");
+    tracing::debug!("Using tests directory: {}", base.display());
+
+    let output_dir = std::path::Path::new(manifest_dir).join("../tests/target/difftest");
+    fs::create_dir_all(&output_dir)?;
+    let output_dir = output_dir
+        .canonicalize()
+        .expect("Failed to canonicalize tests directory");
+    tracing::debug!("Using output directory: {}", output_dir.display());
+
+    let runner = Arc::new(Runner {
+        base_dir: base.clone(),
+        output_dir,
+    });
+
+    let test_cases = collect_test_dirs(&base).expect("Failed to collect test case directories");
+    if test_cases.is_empty() {
+        eprintln!("No valid tests found in {}", base.display());
+        process::exit(1);
+    }
+
+    let trails = test_cases
+        .into_iter()
+        .map(|case| {
+            let runner = runner.clone();
+            Trial::test(case.to_string(), move || Ok(runner.run_test_case(&case)?))
+        })
+        .collect();
+    Ok(trails)
+}