From 6b269f0fcec2abd861f5b7fd93511b08e9d331b4 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Wed, 15 Jul 2026 15:34:54 +0000 Subject: [PATCH 1/2] test(e2e): harden chat checks and CI gating Signed-off-by: Roman Inflianskas --- .github/workflows/ci.yml | 1 + tests/e2e-cucumber/src/lib.rs | 29 ++++++++++++++++++++++ tests/e2e-cucumber/tests/e2e/chat_steps.rs | 11 +++++--- 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7012da2b..337a077f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,6 +116,7 @@ jobs: - '**/*.sh' - '**/*.ps1' - '**/*.feature' + - 'tests/e2e-cucumber/expectations.toml' - 'install*' - '.github/workflows/**' # THIRD_PARTY_NOTICES.txt staleness gate: anything that changes the diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 07471e93..3fcdcec2 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -6,8 +6,37 @@ pub mod capability; pub mod expectation; pub mod mock_server; +pub fn chat_response_is_successful(response: &serde_json::Value) -> bool { + response + .get("choices") + .and_then(serde_json::Value::as_array) + .is_some_and(|choices| !choices.is_empty()) +} + // The report generator lives in its own lean crate (only maud + serde) so xtask // can reuse it without pulling this harness's heavy tree. Re-export it under the // original path so `e2e_cucumber::report::{generate, evaluate_xfail}` call sites // keep working. pub use e2e_report as report; + +#[cfg(test)] +mod tests { + use super::chat_response_is_successful; + + #[test] + fn chat_success_requires_non_empty_choices_array() { + assert!(!chat_response_is_successful(&serde_json::json!({}))); + assert!(!chat_response_is_successful( + &serde_json::json!({"choices": null}) + )); + assert!(!chat_response_is_successful( + &serde_json::json!({"choices": {}}) + )); + assert!(!chat_response_is_successful( + &serde_json::json!({"choices": []}) + )); + assert!(chat_response_is_successful( + &serde_json::json!({"choices": [{}]}) + )); + } +} diff --git a/tests/e2e-cucumber/tests/e2e/chat_steps.rs b/tests/e2e-cucumber/tests/e2e/chat_steps.rs index d2c08cdd..ac92f35c 100644 --- a/tests/e2e-cucumber/tests/e2e/chat_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/chat_steps.rs @@ -54,7 +54,12 @@ async fn send_chat_with_tools(world: &mut E2eWorld) { .to_string(); let chat_url = format!("{endpoint}/chat/completions"); - let client = reqwest::Client::new(); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs( + crate::inference_timeout_for(world), + )) + .build() + .expect("failed to build HTTP client"); let chat_resp: serde_json::Value = client .post(&chat_url) .json(&serde_json::json!({ @@ -152,8 +157,8 @@ async fn assert_privacy_notice_accurate(_world: &mut E2eWorld) { async fn assert_chat_successful(world: &mut E2eWorld) { let resp = world.chat_response.as_ref().expect("no chat response"); assert!( - resp.get("choices").is_some(), - "no choices in response: {resp}" + e2e_cucumber::chat_response_is_successful(resp), + "no non-empty choices array in response: {resp}" ); } From 9702e95e534818b14b05d3ff3d32ccabfd2c416d Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 17 Jul 2026 11:20:06 +0000 Subject: [PATCH 2/2] test(e2e): time out model discovery Signed-off-by: Roman Inflianskas --- tests/e2e-cucumber/tests/e2e/chat_steps.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/chat_steps.rs b/tests/e2e-cucumber/tests/e2e/chat_steps.rs index ac92f35c..b781de12 100644 --- a/tests/e2e-cucumber/tests/e2e/chat_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/chat_steps.rs @@ -41,8 +41,18 @@ async fn user_offered_endpoint(_world: &mut E2eWorld) {} #[when("a chat request with tool definitions is sent")] async fn send_chat_with_tools(world: &mut E2eWorld) { let endpoint = world.endpoint.as_ref().expect("no endpoint configured"); + + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs( + crate::inference_timeout_for(world), + )) + .build() + .expect("failed to build HTTP client"); + let models_url = format!("{endpoint}/models"); - let resp: serde_json::Value = reqwest::get(&models_url) + let resp: serde_json::Value = client + .get(&models_url) + .send() .await .unwrap_or_else(|e| panic!("GET {models_url} failed: {e}")) .json() @@ -54,12 +64,6 @@ async fn send_chat_with_tools(world: &mut E2eWorld) { .to_string(); let chat_url = format!("{endpoint}/chat/completions"); - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs( - crate::inference_timeout_for(world), - )) - .build() - .expect("failed to build HTTP client"); let chat_resp: serde_json::Value = client .post(&chat_url) .json(&serde_json::json!({