feat(harness): Give users control over sort order (#96)

Author: epage

Cargo.lock

@@ -149,3 +149,10 @@ support for multi-character shorts is something people may want to opt-in to (it
 Performance isn't the top priority, so remoing `&str` -> `char` conversions isn't necessarily viewed as a benefit.
 This also makes `match` need to work off of `&str` instead of `char`.
 Unsure which of those would be slower and how the different characteristics match up.
+
+## Harness
+
+### Decision: report and run tests in filter order
+
+Rather than build into every harness shuffle, sharding, and any other specific logic like that,
+we can instead give the user direct control over the test order by the order they are specified on the command line.

DESIGN.md

@@ -161,9 +161,6 @@ fn discover(
     notifier.notify(notify::Event::DiscoverStart)?;
     let timer = std::time::Instant::now();
 
-    // Do this first so it applies to both discover and running
-    cases.sort_unstable_by_key(|case| case.name().to_owned());
-
     let matches_filter = |case: &dyn Case, filter: &str| {
         let test_name = case.name();
 
@@ -172,6 +169,20 @@ fn discover(
             false => test_name.contains(filter),
         }
     };
+
+    // Do this first so it applies to both discover and running
+    cases.sort_unstable_by_key(|case| {
+        let priority = if opts.filters.is_empty() {
+            Some(0)
+        } else {
+            opts.filters
+                .iter()
+                .position(|filter| matches_filter(case.as_ref(), filter))
+        };
+        let name = case.name().to_owned();
+        (priority, name)
+    });
+
     let mut retain_cases = Vec::with_capacity(cases.len());
     for case in cases.iter() {
         let filtered_in = opts.filters.is_empty()

crates/libtest2-harness/src/harness.rs

@@ -37,7 +37,7 @@ dunce = "1.0.4"
 escargot = "0.5.8"
 once_cell_polyfill = "1.56.0"
 pathdiff = "0.2.1"
-snapbox = "0.6.0"
+snapbox = { version = "0.6.0", features = ["json"] }
 
 [lints]
 workspace = true

crates/libtest2-mimic/Cargo.toml

@@ -1,3 +1,6 @@
+use snapbox::prelude::*;
+use snapbox::str;
+
 fn test_cmd() -> snapbox::cmd::Command {
     static BIN: once_cell_polyfill::sync::OnceLock<(std::path::PathBuf, std::path::PathBuf)> =
         once_cell_polyfill::sync::OnceLock::new();
@@ -23,7 +26,7 @@ fn main() {
     snapbox::cmd::Command::new(bin).current_dir(current_dir)
 }
 
-fn check(args: &[&str], single: &str, parallel: &str) {
+fn check(args: &[&str], single: impl IntoData, parallel: impl IntoData) {
     test_cmd()
         .args(args)
         .args(["--test-threads", "1"])
@@ -37,191 +40,227 @@ fn check(args: &[&str], single: &str, parallel: &str) {
 fn normal() {
     check(
         &[],
-        r#"
+        str![[r#"
+
 running 3 tests
 test bar   ... ok
 test barro ... ok
 test foo   ... ok
 
 test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s
 
-"#,
-        r#"
+
+"#]],
+        str![[r#"
+
 running 3 tests
 ...
 
 test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s
 
-"#,
+
+"#]],
     );
 }
 
 #[test]
 fn filter_one() {
     check(
         &["foo"],
-        r#"
+        str![[r#"
+
 running 1 test
 test foo ... ok
 
 test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
 
-"#,
-        r#"
+
+"#]],
+        str![[r#"
+
 running 1 test
 test foo ... ok
 
 test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
 
-"#,
+
+"#]],
     );
 }
 
 #[test]
 fn filter_two() {
     check(
         &["bar"],
-        r#"
+        str![[r#"
+
 running 2 tests
 test bar   ... ok
 test barro ... ok
 
 test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s
 
-"#,
-        r#"
+
+"#]],
+        str![[r#"
+
 running 2 tests
 ...
 
 test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s
 
-"#,
+
+"#]],
     );
 }
 
 #[test]
 fn filter_exact() {
     check(
         &["bar", "--exact"],
-        r#"
+        str![[r#"
+
 running 1 test
 test bar ... ok
 
 test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
 
-"#,
-        r#"
+
+"#]],
+        str![[r#"
+
 running 1 test
 test bar ... ok
 
 test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
 
-"#,
+
+"#]],
     );
 }
 
 #[test]
 fn filter_two_and_skip() {
     check(
         &["--skip", "barro", "bar"],
-        r#"
+        str![[r#"
+
 running 1 test
 test bar ... ok
 
 test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
 
-"#,
-        r#"
+
+"#]],
+        str![[r#"
+
 running 1 test
 test bar ... ok
 
 test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
 
-"#,
+
+"#]],
     );
 }
 
 #[test]
 fn skip_nothing() {
     check(
         &["--skip", "peter"],
-        r#"
+        str![[r#"
+
 running 3 tests
 test bar   ... ok
 test barro ... ok
 test foo   ... ok
 
 test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s
 
-"#,
-        r#"
+
+"#]],
+        str![[r#"
+
 running 3 tests
 ...
 
 test result: ok. 3 passed; 0 failed; 0 ignored; 0 filtered out; finished in [..]s
 
-"#,
+
+"#]],
     );
 }
 
 #[test]
 fn skip_two() {
     check(
         &["--skip", "bar"],
-        r#"
+        str![[r#"
+
 running 1 test
 test foo ... ok
 
 test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
 
-"#,
-        r#"
+
+"#]],
+        str![[r#"
+
 running 1 test
 test foo ... ok
 
 test result: ok. 1 passed; 0 failed; 0 ignored; 2 filtered out; finished in [..]s
 
-"#,
+
+"#]],
     );
 }
 
 #[test]
 fn skip_exact() {
     check(
         &["--exact", "--skip", "bar"],
-        r#"
+        str![[r#"
+
 running 2 tests
 test barro ... ok
 test foo   ... ok
 
 test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s
 
-"#,
-        r#"
+
+"#]],
+        str![[r#"
+
 running 2 tests
 ...
 
 test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s
 
-"#,
+
+"#]],
     );
 }
 
 #[test]
 fn terse_output() {
     check(
         &["--quiet", "--skip", "foo"],
-        r#"
+        str![[r#"
+
 running 2 tests
 ..
 test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s
 
-"#,
-        r#"
+
+"#]],
+        str![[r#"
+
 running 2 tests
 ..
 test result: ok. 2 passed; 0 failed; 0 ignored; 1 filtered out; finished in [..]s
 
-"#,
+
+"#]],
     );
 }

crates/libtest2-mimic/tests/testsuite/all_passing.rs

@@ -1,3 +1,5 @@
+use snapbox::str;
+
 #[test]
 fn check_test_on_main_thread() {
     let package_root = crate::util::new_test(
@@ -22,9 +24,10 @@ fn main() {
         .current_dir(package_root)
         .assert()
         .success()
-        .stdout_eq(
-            "
+        .stdout_eq(str![[r#"
+
 running 1 test
-...",
-        );
+...
+
+"#]]);
 }