Skip to content

Commit 45f0089

Browse files
GuanyiLi-Craigclaude
GuanyiLi-Craig
and
claude
authored
ci: stop swallowing integration test output on failure (#104)
Each integration step ran the test runner under `set -e` with `output=$(uv run python ...)`. When the runner exits non-zero, `set -e` aborts the step at the capture line, so the subsequent `echo "$output"` never runs and all failure detail (failed test names, tracebacks, API errors) is discarded. Logs jump straight from `##[endgroup]` to `exit code 1`, making failures impossible to diagnose. Capture the runner's exit status explicitly (`set +e` around the call), always echo the captured output, and treat a non-zero exit as failure in addition to the existing "Failed:" grep. The runner already returns exit code 1 on any failed example, so this is the more reliable signal. Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent ec97a92 commit 45f0089

1 file changed

Lines changed: 48 additions & 24 deletions

File tree

‎.github/workflows/ci.yaml‎

Lines changed: 48 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -109,10 +109,12 @@ jobs:
109109

110110
- name: Run Integration Tests - agents
111111
run: |
112-
set -e
112+
set +e
113113
output=$(uv run python tests_integration/agents/run_agents.py)
114+
status=$?
115+
set -e
114116
echo "$output"
115-
if echo "$output" | grep -q "Failed:"; then
117+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
116118
echo "### Integration Test Results - agents" >> $GITHUB_STEP_SUMMARY
117119
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
118120
exit 1
@@ -152,10 +154,12 @@ jobs:
152154

153155
- name: Run Integration Tests - embedding_assistant
154156
run: |
155-
set -e
157+
set +e
156158
output=$(uv run python tests_integration/embedding_assistant/run_embedding_assistant.py)
159+
status=$?
160+
set -e
157161
echo "$output"
158-
if echo "$output" | grep -q "Failed:"; then
162+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
159163
echo "### Integration Test Results - embedding_assistant" >> $GITHUB_STEP_SUMMARY
160164
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
161165
exit 1
@@ -195,10 +199,12 @@ jobs:
195199

196200
- name: Run Integration Tests - function_assistant
197201
run: |
198-
set -e
202+
set +e
199203
output=$(uv run python tests_integration/function_assistant/run_function_assistant.py)
204+
status=$?
205+
set -e
200206
echo "$output"
201-
if echo "$output" | grep -q "Failed:"; then
207+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
202208
echo "### Integration Test Results - function_assistant" >> $GITHUB_STEP_SUMMARY
203209
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
204210
exit 1
@@ -238,10 +244,12 @@ jobs:
238244

239245
- name: Run Integration Tests - function_call_assistant
240246
run: |
241-
set -e
247+
set +e
242248
output=$(uv run python tests_integration/function_call_assistant/run_function_call_assistant.py)
249+
status=$?
250+
set -e
243251
echo "$output"
244-
if echo "$output" | grep -q "Failed:"; then
252+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
245253
echo "### Integration Test Results - function_call_assistant" >> $GITHUB_STEP_SUMMARY
246254
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
247255
exit 1
@@ -281,10 +289,12 @@ jobs:
281289

282290
- name: Run Integration Tests - hith_assistant
283291
run: |
284-
set -e
292+
set +e
285293
output=$(uv run python tests_integration/hith_assistant/run_hith_assistant.py)
294+
status=$?
295+
set -e
286296
echo "$output"
287-
if echo "$output" | grep -q "Failed:"; then
297+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
288298
echo "### Integration Test Results - hith_assistant" >> $GITHUB_STEP_SUMMARY
289299
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
290300
exit 1
@@ -324,10 +334,12 @@ jobs:
324334

325335
- name: Run Integration Tests - input_output_topics
326336
run: |
327-
set -e
337+
set +e
328338
output=$(uv run python tests_integration/input_output_topics/run_input_output_topics.py)
339+
status=$?
340+
set -e
329341
echo "$output"
330-
if echo "$output" | grep -q "Failed:"; then
342+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
331343
echo "### Integration Test Results - input_output_topics" >> $GITHUB_STEP_SUMMARY
332344
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
333345
exit 1
@@ -367,10 +379,12 @@ jobs:
367379

368380
- name: Run Integration Tests - invoke_kwargs
369381
run: |
370-
set -e
382+
set +e
371383
output=$(uv run python tests_integration/invoke_kwargs/run_invoke_kwargs.py)
384+
status=$?
385+
set -e
372386
echo "$output"
373-
if echo "$output" | grep -q "Failed:"; then
387+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
374388
echo "### Integration Test Results - invoke_kwargs" >> $GITHUB_STEP_SUMMARY
375389
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
376390
exit 1
@@ -410,10 +424,12 @@ jobs:
410424

411425
- name: Run Integration Tests - multimodal_assistant
412426
run: |
413-
set -e
427+
set +e
414428
output=$(uv run python tests_integration/multimodal_assistant/run_multimodal_assistant.py)
429+
status=$?
430+
set -e
415431
echo "$output"
416-
if echo "$output" | grep -q "Failed:"; then
432+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
417433
echo "### Integration Test Results - multimodal_assistant" >> $GITHUB_STEP_SUMMARY
418434
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
419435
exit 1
@@ -453,10 +469,12 @@ jobs:
453469

454470
- name: Run Integration Tests - rag_assistant
455471
run: |
456-
set -e
472+
set +e
457473
output=$(uv run python tests_integration/rag_assistant/run_rag_assistant.py)
474+
status=$?
475+
set -e
458476
echo "$output"
459-
if echo "$output" | grep -q "Failed:"; then
477+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
460478
echo "### Integration Test Results - rag_assistant" >> $GITHUB_STEP_SUMMARY
461479
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
462480
exit 1
@@ -496,10 +514,12 @@ jobs:
496514

497515
- name: Run Integration Tests - react_assistant
498516
run: |
499-
set -e
517+
set +e
500518
output=$(uv run python tests_integration/react_assistant/run_react_assistant.py)
519+
status=$?
520+
set -e
501521
echo "$output"
502-
if echo "$output" | grep -q "Failed:"; then
522+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
503523
echo "### Integration Test Results - react_assistant" >> $GITHUB_STEP_SUMMARY
504524
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
505525
exit 1
@@ -539,10 +559,12 @@ jobs:
539559

540560
- name: Run Integration Tests - simple_llm_assistant
541561
run: |
542-
set -e
562+
set +e
543563
output=$(uv run python tests_integration/simple_llm_assistant/run_simple_llm_assistant.py)
564+
status=$?
565+
set -e
544566
echo "$output"
545-
if echo "$output" | grep -q "Failed:"; then
567+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
546568
echo "### Integration Test Results - simple_llm_assistant" >> $GITHUB_STEP_SUMMARY
547569
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
548570
exit 1
@@ -582,10 +604,12 @@ jobs:
582604

583605
- name: Run Integration Tests - simple_stream_assistant
584606
run: |
585-
set -e
607+
set +e
586608
output=$(uv run python tests_integration/simple_stream_assistant/run_simple_stream_assistant.py)
609+
status=$?
610+
set -e
587611
echo "$output"
588-
if echo "$output" | grep -q "Failed:"; then
612+
if [ "$status" -ne 0 ] || echo "$output" | grep -q "Failed:"; then
589613
echo "### Integration Test Results - simple_stream_assistant" >> $GITHUB_STEP_SUMMARY
590614
echo "❌ Tests failed!" >> $GITHUB_STEP_SUMMARY
591615
exit 1

0 commit comments

Comments
 (0)