diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7012da2b..337a077f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,6 +116,7 @@ jobs: - '**/*.sh' - '**/*.ps1' - '**/*.feature' + - 'tests/e2e-cucumber/expectations.toml' - 'install*' - '.github/workflows/**' # THIRD_PARTY_NOTICES.txt staleness gate: anything that changes the diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 07471e93..3fcdcec2 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -6,8 +6,37 @@ pub mod capability; pub mod expectation; pub mod mock_server; +pub fn chat_response_is_successful(response: &serde_json::Value) -> bool { + response + .get("choices") + .and_then(serde_json::Value::as_array) + .is_some_and(|choices| !choices.is_empty()) +} + // The report generator lives in its own lean crate (only maud + serde) so xtask // can reuse it without pulling this harness's heavy tree. Re-export it under the // original path so `e2e_cucumber::report::{generate, evaluate_xfail}` call sites // keep working. pub use e2e_report as report; + +#[cfg(test)] +mod tests { + use super::chat_response_is_successful; + + #[test] + fn chat_success_requires_non_empty_choices_array() { + assert!(!chat_response_is_successful(&serde_json::json!({}))); + assert!(!chat_response_is_successful( + &serde_json::json!({"choices": null}) + )); + assert!(!chat_response_is_successful( + &serde_json::json!({"choices": {}}) + )); + assert!(!chat_response_is_successful( + &serde_json::json!({"choices": []}) + )); + assert!(chat_response_is_successful( + &serde_json::json!({"choices": [{}]}) + )); + } +} diff --git a/tests/e2e-cucumber/tests/e2e/chat_steps.rs b/tests/e2e-cucumber/tests/e2e/chat_steps.rs index d2c08cdd..b781de12 100644 --- a/tests/e2e-cucumber/tests/e2e/chat_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/chat_steps.rs @@ -41,8 +41,18 @@ async fn user_offered_endpoint(_world: &mut E2eWorld) {} #[when("a chat request with tool definitions is sent")] async fn send_chat_with_tools(world: &mut E2eWorld) { let endpoint = world.endpoint.as_ref().expect("no endpoint configured"); + + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs( + crate::inference_timeout_for(world), + )) + .build() + .expect("failed to build HTTP client"); + let models_url = format!("{endpoint}/models"); - let resp: serde_json::Value = reqwest::get(&models_url) + let resp: serde_json::Value = client + .get(&models_url) + .send() .await .unwrap_or_else(|e| panic!("GET {models_url} failed: {e}")) .json() @@ -54,7 +64,6 @@ async fn send_chat_with_tools(world: &mut E2eWorld) { .to_string(); let chat_url = format!("{endpoint}/chat/completions"); - let client = reqwest::Client::new(); let chat_resp: serde_json::Value = client .post(&chat_url) .json(&serde_json::json!({ @@ -152,8 +161,8 @@ async fn assert_privacy_notice_accurate(_world: &mut E2eWorld) { async fn assert_chat_successful(world: &mut E2eWorld) { let resp = world.chat_response.as_ref().expect("no chat response"); assert!( - resp.get("choices").is_some(), - "no choices in response: {resp}" + e2e_cucumber::chat_response_is_successful(resp), + "no non-empty choices array in response: {resp}" ); }