diff --git a/DESIGN.md b/DESIGN.md index 6bfb3b0..2192e54 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -156,3 +156,15 @@ Unsure which of those would be slower and how the different characteristics matc Rather than build into every harness shuffle, sharding, and any other specific logic like that, we can instead give the user direct control over the test order by the order they are specified on the command line. + +### Decision: argfile support + +Similar to filters changing the order of tests, +argfile support allows for passing a large list of arguments to a test binary. + +The syntax and semantics match rustc: +- Expanded before parsing, independent of any other syntax +- Arguments are delimited by newlines; no shell escaping + - rustc has unstable support for `@shell:` +- Lines are read literal, empty lines are empty arguments and no comments +- Non-recursive diff --git a/crates/libtest2-harness/src/harness.rs b/crates/libtest2-harness/src/harness.rs index 3abc016..432cd97 100644 --- a/crates/libtest2-harness/src/harness.rs +++ b/crates/libtest2-harness/src/harness.rs @@ -3,18 +3,19 @@ use libtest_lexarg::OutputFormat; use crate::{cli, notify, Case, RunError, RunMode, State}; pub struct Harness { - raw: Vec, + raw: std::io::Result>, cases: Vec>, } impl Harness { pub fn with_args(args: impl IntoIterator>) -> Self { - let raw = args.into_iter().map(|s| s.into()).collect::>(); + let raw = expand_args(args); Self { raw, cases: vec![] } } pub fn with_env() -> Self { - let raw = std::env::args_os().collect::>(); + let raw = std::env::args_os(); + let raw = expand_args(raw); Self { raw, cases: vec![] } } @@ -31,7 +32,14 @@ impl Harness { } pub fn main(mut self) -> ! { - let mut parser = cli::Parser::new(&self.raw); + let raw = match self.raw { + Ok(raw) => raw, + Err(err) => { + eprintln!("{err}"); + std::process::exit(1) + } + }; + let mut parser = cli::Parser::new(&raw); let opts = parse(&mut parser).unwrap_or_else(|err| { eprintln!("{err}"); std::process::exit(1) @@ -131,6 +139,27 @@ fn parse<'p>( Ok(opts) } +fn expand_args( + args: impl IntoIterator>, +) -> std::io::Result> { + let mut expanded = Vec::new(); + for arg in args { + let arg = arg.into(); + if let Some(argfile) = arg.to_str().and_then(|s| s.strip_prefix("@")) { + expanded.extend(parse_argfile(std::path::Path::new(argfile))?); + } else { + expanded.push(arg); + } + } + Ok(expanded) +} + +fn parse_argfile(path: &std::path::Path) -> std::io::Result> { + // Logic taken from rust-lang/rust's `compiler/rustc_driver_impl/src/args.rs` + let content = std::fs::read_to_string(path)?; + Ok(content.lines().map(|s| s.into()).collect()) +} + fn notifier(opts: &libtest_lexarg::TestOpts) -> std::io::Result> { #[cfg(feature = "color")] let stdout = anstream::stdout(); diff --git a/crates/libtest2-mimic/tests/testsuite/argfile.rs b/crates/libtest2-mimic/tests/testsuite/argfile.rs new file mode 100644 index 0000000..8d671db --- /dev/null +++ b/crates/libtest2-mimic/tests/testsuite/argfile.rs @@ -0,0 +1,184 @@ +use snapbox::prelude::*; +use snapbox::str; + +fn test_cmd() -> snapbox::cmd::Command { + static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = + once_cell_polyfill::sync::OnceLock::new(); + let (bin, current_dir) = BIN.get_or_init(|| { + let package_root = crate::util::new_test( + r#" +fn main() { + use libtest2_mimic::Trial; + use libtest2_mimic::RunError; + libtest2_mimic::Harness::with_env() + .cases(vec![ + Trial::test("one", |_| Ok(())), + Trial::test("two", |_| Ok(())), + Trial::test("three", |_| Ok(())), + Trial::test("one_two", |_| Ok(())), + ]) + .main(); +} +"#, + false, + ); + let bin = crate::util::compile_test(&package_root); + (bin, package_root) + }); + snapbox::cmd::Command::new(bin).current_dir(current_dir) +} + +fn check( + args: &[&str], + argfile: &std::path::Path, + code: i32, + single: impl IntoData, + parallel: impl IntoData, +) { + test_cmd() + .arg(format!("@{}", argfile.to_str().unwrap())) + .args(args) + .args(["--test-threads", "1"]) + .assert() + .code(code) + .stdout_eq(single); + test_cmd() + .arg(format!("@{}", argfile.to_str().unwrap())) + .args(args) + .assert() + .code(code) + .stdout_eq(parallel); +} + +#[test] +fn empty() { + let argfile = crate::util::new_file("argfile-", ".txt", ""); + check( + &[], + &argfile, + 0, + str![[r#" + +running 4 tests +test one ... ok +test one_two ... ok +test three ... ok +test two ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s + + +"#]], + str![[r#" + +running 4 tests +... + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s + + +"#]], + ); +} + +#[test] +fn list() { + let argfile = crate::util::new_file("argfile-", ".txt", "--list"); + check( + &[], + &argfile, + 0, + str![[r#" +one: test +one_two: test +three: test +two: test + +4 tests + + +"#]], + str![[r#" +one: test +one_two: test +... + +4 tests + + +"#]], + ); +} + +#[test] +fn multiline() { + let argfile = crate::util::new_file( + "argfile-", + ".txt", + "one +two +--exact", + ); + check( + &[], + &argfile, + 0, + str![[r#" + +running 2 tests +test one ... ok +test two ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s + + +"#]], + str![[r#" + +running 2 tests +... + +test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s + + +"#]], + ); +} + +#[test] +fn mixed() { + let argfile = crate::util::new_file( + "argfile-", ".txt", "one +two", + ); + check( + &["--exact"], + &argfile, + 0, + str![[r#" + +running 2 tests +test one ... ok +test two ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s + + +"#]], + str![[r#" + +running 2 tests +... + +test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s + + +"#]], + ); +} + +#[test] +fn invalid() { + let argfile = std::path::Path::new("highly-improbably-non-existent-file.txt"); + check(&[], argfile, 1, str![""], str![""]); +} diff --git a/crates/libtest2-mimic/tests/testsuite/main.rs b/crates/libtest2-mimic/tests/testsuite/main.rs index 1a8e41c..660e056 100644 --- a/crates/libtest2-mimic/tests/testsuite/main.rs +++ b/crates/libtest2-mimic/tests/testsuite/main.rs @@ -1,4 +1,5 @@ mod all_passing; +mod argfile; mod main_thread; mod mixed_bag; mod panic; diff --git a/crates/libtest2-mimic/tests/testsuite/util.rs b/crates/libtest2-mimic/tests/testsuite/util.rs index 993777e..1936bb3 100644 --- a/crates/libtest2-mimic/tests/testsuite/util.rs +++ b/crates/libtest2-mimic/tests/testsuite/util.rs @@ -40,6 +40,15 @@ harness = {harness} package_root } +pub fn new_file(name_prefix: &str, name_suffix: &str, content: &str) -> std::path::PathBuf { + static SUFFIX: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let suffix = SUFFIX.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let name = format!("{name_prefix}{suffix}{name_suffix}"); + let path = tempdir().join(name); + std::fs::write(&path, content).unwrap(); + path +} + pub fn compile_test(package_root: &std::path::Path) -> std::path::PathBuf { let manifest_path = package_root.join("Cargo.toml"); let target_name = package_root.file_name().unwrap().to_str().unwrap(); diff --git a/crates/libtest2/tests/testsuite/argfile.rs b/crates/libtest2/tests/testsuite/argfile.rs new file mode 100644 index 0000000..5850833 --- /dev/null +++ b/crates/libtest2/tests/testsuite/argfile.rs @@ -0,0 +1,189 @@ +use snapbox::prelude::*; +use snapbox::str; + +fn test_cmd() -> snapbox::cmd::Command { + static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> = + once_cell_polyfill::sync::OnceLock::new(); + let (bin, current_dir) = BIN.get_or_init(|| { + let package_root = crate::util::new_test( + r#" +libtest2::libtest2_main!(one, two, three, one_two); + +fn one(_state: &libtest2::State) -> libtest2::RunResult { + Ok(()) +} + +fn two(_state: &libtest2::State) -> libtest2::RunResult { + Ok(()) +} + +fn three(_state: &libtest2::State) -> libtest2::RunResult { + Ok(()) +} + +fn one_two(_state: &libtest2::State) -> libtest2::RunResult { + Ok(()) +} +"#, + false, + ); + let bin = crate::util::compile_test(&package_root); + (bin, package_root) + }); + snapbox::cmd::Command::new(bin).current_dir(current_dir) +} + +fn check( + args: &[&str], + argfile: &std::path::Path, + code: i32, + single: impl IntoData, + parallel: impl IntoData, +) { + test_cmd() + .arg(format!("@{}", argfile.to_str().unwrap())) + .args(args) + .args(["--test-threads", "1"]) + .assert() + .code(code) + .stdout_eq(single); + test_cmd() + .arg(format!("@{}", argfile.to_str().unwrap())) + .args(args) + .assert() + .code(code) + .stdout_eq(parallel); +} + +#[test] +fn empty() { + let argfile = crate::util::new_file("argfile-", ".txt", ""); + check( + &[], + &argfile, + 0, + str![[r#" + +running 4 tests +test one ... ok +test one_two ... ok +test three ... ok +test two ... ok + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s + + +"#]], + str![[r#" + +running 4 tests +... + +test result: ok. 4 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s + + +"#]], + ); +} + +#[test] +fn list() { + let argfile = crate::util::new_file("argfile-", ".txt", "--list"); + check( + &[], + &argfile, + 0, + str![[r#" +one: test +one_two: test +three: test +two: test + +4 tests + + +"#]], + str![[r#" +one: test +one_two: test +... + +4 tests + + +"#]], + ); +} + +#[test] +fn multiline() { + let argfile = crate::util::new_file( + "argfile-", + ".txt", + "one +two +--exact", + ); + check( + &[], + &argfile, + 0, + str![[r#" + +running 2 tests +test one ... ok +test two ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s + + +"#]], + str![[r#" + +running 2 tests +... + +test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s + + +"#]], + ); +} + +#[test] +fn mixed() { + let argfile = crate::util::new_file( + "argfile-", ".txt", "one +two", + ); + check( + &["--exact"], + &argfile, + 0, + str![[r#" + +running 2 tests +test one ... ok +test two ... ok + +test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s + + +"#]], + str![[r#" + +running 2 tests +... + +test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s + + +"#]], + ); +} + +#[test] +fn invalid() { + let argfile = std::path::Path::new("highly-improbably-non-existent-file.txt"); + check(&[], argfile, 1, str![""], str![""]); +} diff --git a/crates/libtest2/tests/testsuite/main.rs b/crates/libtest2/tests/testsuite/main.rs index d74b0f5..66bad57 100644 --- a/crates/libtest2/tests/testsuite/main.rs +++ b/crates/libtest2/tests/testsuite/main.rs @@ -1,4 +1,5 @@ mod all_passing; +mod argfile; mod mixed_bag; mod panic; mod util; diff --git a/crates/libtest2/tests/testsuite/util.rs b/crates/libtest2/tests/testsuite/util.rs index 158d58a..5558e0f 100644 --- a/crates/libtest2/tests/testsuite/util.rs +++ b/crates/libtest2/tests/testsuite/util.rs @@ -40,6 +40,15 @@ harness = {harness} package_root } +pub fn new_file(name_prefix: &str, name_suffix: &str, content: &str) -> std::path::PathBuf { + static SUFFIX: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0); + let suffix = SUFFIX.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let name = format!("{name_prefix}{suffix}{name_suffix}"); + let path = tempdir().join(name); + std::fs::write(&path, content).unwrap(); + path +} + pub fn compile_test(package_root: &std::path::Path) -> std::path::PathBuf { let manifest_path = package_root.join("Cargo.toml"); let target_name = package_root.file_name().unwrap().to_str().unwrap();