feat(harness): Add argfile support (#98)

Author: epage

DESIGN.md

@@ -156,3 +156,15 @@ Unsure which of those would be slower and how the different characteristics matc
 
 Rather than build into every harness shuffle, sharding, and any other specific logic like that,
 we can instead give the user direct control over the test order by the order they are specified on the command line.
+
+### Decision: argfile support
+
+Similar to filters changing the order of tests,
+argfile support allows for passing a large list of arguments to a test binary.
+
+The syntax and semantics match rustc:
+- Expanded before parsing, independent of any other syntax
+- Arguments are delimited by newlines; no shell escaping
+  - rustc has unstable support for `@shell:<path>`
+- Lines are read literal, empty lines are empty arguments and no comments
+- Non-recursive

crates/libtest2-harness/src/harness.rs

@@ -3,18 +3,19 @@ use libtest_lexarg::OutputFormat;
 use crate::{cli, notify, Case, RunError, RunMode, State};
 
 pub struct Harness {
-    raw: Vec<std::ffi::OsString>,
+    raw: std::io::Result<Vec<std::ffi::OsString>>,
     cases: Vec<Box<dyn Case>>,
 }
 
 impl Harness {
     pub fn with_args(args: impl IntoIterator<Item = impl Into<std::ffi::OsString>>) -> Self {
-        let raw = args.into_iter().map(|s| s.into()).collect::<Vec<_>>();
+        let raw = expand_args(args);
         Self { raw, cases: vec![] }
     }
 
     pub fn with_env() -> Self {
-        let raw = std::env::args_os().collect::<Vec<_>>();
+        let raw = std::env::args_os();
+        let raw = expand_args(raw);
         Self { raw, cases: vec![] }
     }
 
@@ -31,7 +32,14 @@ impl Harness {
     }
 
     pub fn main(mut self) -> ! {
-        let mut parser = cli::Parser::new(&self.raw);
+        let raw = match self.raw {
+            Ok(raw) => raw,
+            Err(err) => {
+                eprintln!("{err}");
+                std::process::exit(1)
+            }
+        };
+        let mut parser = cli::Parser::new(&raw);
         let opts = parse(&mut parser).unwrap_or_else(|err| {
             eprintln!("{err}");
             std::process::exit(1)
@@ -131,6 +139,27 @@ fn parse<'p>(
     Ok(opts)
 }
 
+fn expand_args(
+    args: impl IntoIterator<Item = impl Into<std::ffi::OsString>>,
+) -> std::io::Result<Vec<std::ffi::OsString>> {
+    let mut expanded = Vec::new();
+    for arg in args {
+        let arg = arg.into();
+        if let Some(argfile) = arg.to_str().and_then(|s| s.strip_prefix("@")) {
+            expanded.extend(parse_argfile(std::path::Path::new(argfile))?);
+        } else {
+            expanded.push(arg);
+        }
+    }
+    Ok(expanded)
+}
+
+fn parse_argfile(path: &std::path::Path) -> std::io::Result<Vec<std::ffi::OsString>> {
+    // Logic taken from rust-lang/rust's `compiler/rustc_driver_impl/src/args.rs`
+    let content = std::fs::read_to_string(path)?;
+    Ok(content.lines().map(|s| s.into()).collect())
+}
+
 fn notifier(opts: &libtest_lexarg::TestOpts) -> std::io::Result<Box<dyn notify::Notifier>> {
     #[cfg(feature = "color")]
     let stdout = anstream::stdout();

crates/libtest2-mimic/tests/testsuite/argfile.rs

@@ -0,0 +1,184 @@
+use snapbox::prelude::*;
+use snapbox::str;
+
+fn test_cmd() -> snapbox::cmd::Command {
+    static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> =
+        once_cell_polyfill::sync::OnceLock::new();
+    let (bin, current_dir) = BIN.get_or_init(|| {
+        let package_root = crate::util::new_test(
+            r#"
+fn main() {
+    use libtest2_mimic::Trial;
+    use libtest2_mimic::RunError;
+    libtest2_mimic::Harness::with_env()
+        .cases(vec![
+            Trial::test("one", |_| Ok(())),
+            Trial::test("two", |_| Ok(())),
+            Trial::test("three", |_| Ok(())),
+            Trial::test("one_two", |_| Ok(())),
+        ])
+        .main();
+}
+"#,
+            false,
+        );
+        let bin = crate::util::compile_test(&package_root);
+        (bin, package_root)
+    });
+    snapbox::cmd::Command::new(bin).current_dir(current_dir)
+}
+
+fn check(
+    args: &[&str],
+    argfile: &std::path::Path,
+    code: i32,
+    single: impl IntoData,
+    parallel: impl IntoData,
+) {
+    test_cmd()
+        .arg(format!("@{}", argfile.to_str().unwrap()))
+        .args(args)
+        .args(["--test-threads", "1"])
+        .assert()
+        .code(code)
+        .stdout_eq(single);
+    test_cmd()
+        .arg(format!("@{}", argfile.to_str().unwrap()))
+        .args(args)
+        .assert()
+        .code(code)
+        .stdout_eq(parallel);
+}
+
+#[test]
+fn empty() {
+    let argfile = crate::util::new_file("argfile-", ".txt", "");
+    check(
+        &[],
+        &argfile,
+        0,
+        str![[r#"
+
+running 4 tests
+test one     ... ok
+test one_two ... ok
+test three   ... ok
+test two     ... ok
+
+test result: ok. 4 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s
+
+
+"#]],
+        str![[r#"
+
+running 4 tests
+...
+
+test result: ok. 4 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s
+
+
+"#]],
+    );
+}
+
+#[test]
+fn list() {
+    let argfile = crate::util::new_file("argfile-", ".txt", "--list");
+    check(
+        &[],
+        &argfile,
+        0,
+        str![[r#"
+one: test
+one_two: test
+three: test
+two: test
+
+4 tests
+
+
+"#]],
+        str![[r#"
+one: test
+one_two: test
+...
+
+4 tests
+
+
+"#]],
+    );
+}
+
+#[test]
+fn multiline() {
+    let argfile = crate::util::new_file(
+        "argfile-",
+        ".txt",
+        "one
+two
+--exact",
+    );
+    check(
+        &[],
+        &argfile,
+        0,
+        str![[r#"
+
+running 2 tests
+test one ... ok
+test two ... ok
+
+test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
+
+
+"#]],
+        str![[r#"
+
+running 2 tests
+...
+
+test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
+
+
+"#]],
+    );
+}
+
+#[test]
+fn mixed() {
+    let argfile = crate::util::new_file(
+        "argfile-", ".txt", "one
+two",
+    );
+    check(
+        &["--exact"],
+        &argfile,
+        0,
+        str![[r#"
+
+running 2 tests
+test one ... ok
+test two ... ok
+
+test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
+
+
+"#]],
+        str![[r#"
+
+running 2 tests
+...
+
+test result: ok. 2 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
+
+
+"#]],
+    );
+}
+
+#[test]
+fn invalid() {
+    let argfile = std::path::Path::new("highly-improbably-non-existent-file.txt");
+    check(&[], argfile, 1, str![""], str![""]);
+}

crates/libtest2-mimic/tests/testsuite/main.rs

@@ -1,4 +1,5 @@
 mod all_passing;
+mod argfile;
 mod main_thread;
 mod mixed_bag;
 mod panic;

crates/libtest2-mimic/tests/testsuite/util.rs

@@ -40,6 +40,15 @@ harness = {harness}
     package_root
 }
 
+pub fn new_file(name_prefix: &str, name_suffix: &str, content: &str) -> std::path::PathBuf {
+    static SUFFIX: std::sync::atomic::AtomicU32 = std::sync::atomic::AtomicU32::new(0);
+    let suffix = SUFFIX.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+    let name = format!("{name_prefix}{suffix}{name_suffix}");
+    let path = tempdir().join(name);
+    std::fs::write(&path, content).unwrap();
+    path
+}
+
 pub fn compile_test(package_root: &std::path::Path) -> std::path::PathBuf {
     let manifest_path = package_root.join("Cargo.toml");
     let target_name = package_root.file_name().unwrap().to_str().unwrap();