feat(json)!: Add CaseMessage

epage · epage · commit 49f8e8d99b7a · 2025-08-20T14:29:02.000-05:00
The aim is to allow multiple failures
diff --git a/DESIGN.md b/DESIGN.md
@@ -37,6 +37,7 @@ Decisions
   - Terse and pretty progress indicators are too nebulous to render (see their notifiers)
   - There is likely not enough value add in the failure message
   - This puts more of a burden on custom test harnesses for their implementation than is strictly needed
+- Report failures separate from test-complete so we can have multiple
 
 ### Prior Art
 
diff --git a/crates/libtest-json/event.schema.json b/crates/libtest-json/event.schema.json
@@ -67,6 +67,19 @@
         "event"
       ]
     },
+    {
+      "type": "object",
+      "properties": {
+        "event": {
+          "type": "string",
+          "const": "case_message"
+        }
+      },
+      "$ref": "#/$defs/CaseMessage",
+      "required": [
+        "event"
+      ]
+    },
     {
       "type": "object",
       "properties": {
@@ -207,6 +220,37 @@
         "failed"
       ]
     },
+    "CaseMessage": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string"
+        },
+        "status": {
+          "$ref": "#/$defs/RunStatus"
+        },
+        "message": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "elapsed_s": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/Elapsed"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        }
+      },
+      "required": [
+        "name",
+        "status"
+      ]
+    },
     "CaseComplete": {
       "type": "object",
       "properties": {
diff --git a/crates/libtest-json/src/event.rs b/crates/libtest-json/src/event.rs
@@ -9,6 +9,7 @@ pub enum Event {
     DiscoverComplete(DiscoverComplete),
     RunStart(RunStart),
     CaseStart(CaseStart),
+    CaseMessage(CaseMessage),
     CaseComplete(CaseComplete),
     RunComplete(RunComplete),
 }
@@ -22,6 +23,7 @@ impl Event {
             Self::DiscoverComplete(event) => event.to_jsonline(),
             Self::RunStart(event) => event.to_jsonline(),
             Self::CaseStart(event) => event.to_jsonline(),
+            Self::CaseMessage(event) => event.to_jsonline(),
             Self::CaseComplete(event) => event.to_jsonline(),
             Self::RunComplete(event) => event.to_jsonline(),
         }
@@ -58,6 +60,12 @@ impl From<CaseStart> for Event {
     }
 }
 
+impl From<CaseMessage> for Event {
+    fn from(inner: CaseMessage) -> Self {
+        Self::CaseMessage(inner)
+    }
+}
+
 impl From<CaseComplete> for Event {
     fn from(inner: CaseComplete) -> Self {
         Self::CaseComplete(inner)
@@ -292,6 +300,67 @@ impl CaseStart {
     }
 }
 
+#[derive(Clone, Debug)]
+#[cfg_attr(feature = "unstable-schema", derive(schemars::JsonSchema))]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
+#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))]
+pub struct CaseMessage {
+    pub name: String,
+    pub status: RunStatus,
+    #[cfg_attr(
+        feature = "serde",
+        serde(default, skip_serializing_if = "Option::is_none")
+    )]
+    pub message: Option<String>,
+    #[cfg_attr(
+        feature = "serde",
+        serde(default, skip_serializing_if = "Option::is_none")
+    )]
+    pub elapsed_s: Option<Elapsed>,
+}
+
+impl CaseMessage {
+    #[cfg(feature = "json")]
+    pub fn to_jsonline(&self) -> String {
+        use json_write::JsonWrite as _;
+
+        let mut buffer = String::new();
+        buffer.open_object().unwrap();
+
+        buffer.key("event").unwrap();
+        buffer.keyval_sep().unwrap();
+        buffer.value("case_message").unwrap();
+
+        buffer.val_sep().unwrap();
+        buffer.key("name").unwrap();
+        buffer.keyval_sep().unwrap();
+        buffer.value(&self.name).unwrap();
+
+        buffer.val_sep().unwrap();
+        buffer.key("status").unwrap();
+        buffer.keyval_sep().unwrap();
+        buffer.value(self.status.as_str()).unwrap();
+
+        if let Some(message) = &self.message {
+            buffer.val_sep().unwrap();
+            buffer.key("message").unwrap();
+            buffer.keyval_sep().unwrap();
+            buffer.value(message).unwrap();
+        }
+
+        if let Some(elapsed_s) = self.elapsed_s {
+            buffer.val_sep().unwrap();
+            buffer.key("elapsed_s").unwrap();
+            buffer.keyval_sep().unwrap();
+            buffer.value(String::from(elapsed_s)).unwrap();
+        }
+
+        buffer.close_object().unwrap();
+
+        buffer
+    }
+}
+
 #[derive(Clone, Debug)]
 #[cfg_attr(feature = "unstable-schema", derive(schemars::JsonSchema))]
 #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
diff --git a/crates/libtest-json/tests/roundtrip.rs b/crates/libtest-json/tests/roundtrip.rs
@@ -101,6 +101,31 @@ fn case_start() {
     );
 }
 
+#[test]
+fn case_message() {
+    t(
+        libtest_json::event::CaseMessage {
+            name: "Hello\tworld!".to_owned(),
+            status: libtest_json::RunStatus::Failed,
+            message: None,
+            elapsed_s: None,
+        },
+        str![[r#"{"event":"case_message","name":"Hello\tworld!","status":"failed"}"#]],
+    );
+
+    t(
+        libtest_json::event::CaseMessage {
+            name: "Hello\tworld!".to_owned(),
+            status: libtest_json::RunStatus::Ignored,
+            message: Some("This\tfailed".to_owned()),
+            elapsed_s: Some(libtest_json::Elapsed(Default::default())),
+        },
+        str![[
+            r#"{"event":"case_message","name":"Hello\tworld!","status":"ignored","message":"This\tfailed","elapsed_s":"0"}"#
+        ]],
+    );
+}
+
 #[test]
 fn case_complete() {
     t(
diff --git a/crates/libtest2-harness/src/notify/pretty.rs b/crates/libtest2-harness/src/notify/pretty.rs
@@ -51,6 +51,7 @@ impl<W: std::io::Write> super::Notifier for PrettyRunNotifier<W> {
                     self.writer.flush()?;
                 }
             }
+            Event::CaseMessage(_) => {}
             Event::CaseComplete(inner) => {
                 let status = self.summary.get_status(&inner.name);
                 let (s, style) = match status {
diff --git a/crates/libtest2-harness/src/notify/summary.rs b/crates/libtest2-harness/src/notify/summary.rs
@@ -1,4 +1,4 @@
-use super::event::CaseComplete;
+use super::event::CaseMessage;
 use super::Event;
 use super::RunStatus;
 use super::FAILED;
@@ -11,14 +11,14 @@ pub(crate) struct Summary {
     /// filter-in pattern or by `--skip` arguments).
     num_filtered_out: usize,
 
-    status: std::collections::HashMap<String, CaseComplete>,
+    status: std::collections::HashMap<String, CaseStatus>,
     elapsed_s: Option<super::Elapsed>,
 }
 
 impl Summary {
     pub(crate) fn get_status(&self, name: &str) -> Option<RunStatus> {
-        let event = self.status.get(name)?;
-        event.status
+        let status = self.status.get(name)?;
+        find_run_status(status)
     }
 
     pub(crate) fn write_start(&self, writer: &mut dyn std::io::Write) -> std::io::Result<()> {
@@ -34,12 +34,27 @@ impl Summary {
         let mut num_failed = 0;
         let mut num_ignored = 0;
         let mut failures = std::collections::BTreeMap::new();
-        for event in self.status.values() {
-            match event.status {
+        for (name, case_status) in &self.status {
+            let mut status = find_run_status(case_status);
+            if !case_status.started {
+                // Even override `Ignored`
+                status = Some(RunStatus::Failed);
+                failures.insert(name, Some("test found that never started"));
+            }
+            if !case_status.completed {
+                // Even override `Ignored`
+                status = Some(RunStatus::Failed);
+                failures.insert(name, Some("test never completed"));
+            }
+            match status {
                 Some(RunStatus::Ignored) => num_ignored += 1,
                 Some(RunStatus::Failed) => {
                     num_failed += 1;
-                    failures.insert(&event.name, &event.message);
+                    for event in &case_status.messages {
+                        if Some(event.status) == status {
+                            failures.insert(name, event.message.as_deref());
+                        }
+                    }
                 }
                 None => num_passed += 1,
             }
@@ -106,9 +121,30 @@ impl super::Notifier for Summary {
             }
             Event::DiscoverComplete(_) => {}
             Event::RunStart(_) => {}
-            Event::CaseStart(_) => {}
+            Event::CaseStart(inner) => {
+                self.status.entry(inner.name).or_default().started = true;
+            }
+            Event::CaseMessage(inner) => {
+                self.status
+                    .entry(inner.name.clone())
+                    .or_default()
+                    .messages
+                    .push(inner);
+            }
             Event::CaseComplete(inner) => {
-                self.status.insert(inner.name.clone(), inner);
+                if let Some(status) = inner.status {
+                    self.status
+                        .entry(inner.name.clone())
+                        .or_default()
+                        .messages
+                        .push(CaseMessage {
+                            name: inner.name.clone(),
+                            status,
+                            message: inner.message.clone(),
+                            elapsed_s: inner.elapsed_s,
+                        });
+                }
+                self.status.entry(inner.name).or_default().completed = true;
             }
             Event::RunComplete(inner) => {
                 self.elapsed_s = inner.elapsed_s;
@@ -117,3 +153,23 @@ impl super::Notifier for Summary {
         Ok(())
     }
 }
+
+fn find_run_status(case_status: &CaseStatus) -> Option<RunStatus> {
+    let mut status = None;
+    for event in &case_status.messages {
+        status = match (status, event.status) {
+            (None, _) => Some(event.status),
+            (Some(RunStatus::Ignored), _) => status,
+            (_, RunStatus::Ignored) => Some(event.status),
+            (Some(RunStatus::Failed), _) => status,
+        }
+    }
+    status
+}
+
+#[derive(Default, Clone, Debug)]
+struct CaseStatus {
+    messages: Vec<CaseMessage>,
+    started: bool,
+    completed: bool,
+}
diff --git a/crates/libtest2-harness/src/notify/terse.rs b/crates/libtest2-harness/src/notify/terse.rs
@@ -35,6 +35,7 @@ impl<W: std::io::Write> super::Notifier for TerseListNotifier<W> {
             }
             Event::RunStart(_) => {}
             Event::CaseStart(_) => {}
+            Event::CaseMessage(_) => {}
             Event::CaseComplete(_) => {}
             Event::RunComplete(_) => {}
         }
@@ -68,6 +69,7 @@ impl<W: std::io::Write> super::Notifier for TerseRunNotifier<W> {
                 self.summary.write_start(&mut self.writer)?;
             }
             Event::CaseStart(_) => {}
+            Event::CaseMessage(_) => {}
             Event::CaseComplete(inner) => {
                 let status = self.summary.get_status(&inner.name);
                 let (c, style) = match status {

Original file line number	Diff line number	Diff line change
`@@ -51,6 +51,7 @@ impl<W: std::io::Write> super::Notifier for PrettyRunNotifier<W> {`
`51`	`51`	`self.writer.flush()?;`
`52`	`52`	`}`
`53`	`53`	`}`
	`54`	`+ Event::CaseMessage(_) => {}`
`54`	`55`	`Event::CaseComplete(inner) => {`
`55`	`56`	`let status = self.summary.get_status(&inner.name);`
`56`	`57`	`let (s, style) = match status {`
Original file line number	Diff line number	Diff line change
`@@ -35,6 +35,7 @@ impl<W: std::io::Write> super::Notifier for TerseListNotifier<W> {`
`35`	`35`	`}`
`36`	`36`	`Event::RunStart(_) => {}`
`37`	`37`	`Event::CaseStart(_) => {}`
	`38`	`+ Event::CaseMessage(_) => {}`
`38`	`39`	`Event::CaseComplete(_) => {}`
`39`	`40`	`Event::RunComplete(_) => {}`
`40`	`41`	`}`
`@@ -68,6 +69,7 @@ impl<W: std::io::Write> super::Notifier for TerseRunNotifier<W> {`
`68`	`69`	`self.summary.write_start(&mut self.writer)?;`
`69`	`70`	`}`
`70`	`71`	`Event::CaseStart(_) => {}`
	`72`	`+ Event::CaseMessage(_) => {}`
`71`	`73`	`Event::CaseComplete(inner) => {`
`72`	`74`	`let status = self.summary.get_status(&inner.name);`
`73`	`75`	`let (c, style) = match status {`