Skip to content

Commit d691175

Browse files
chrisburrchrisburr
authored
fix: improve pytest-integration debugging output (#740)
- Extract debug collection to reusable script - Add collapsible groups for better organization - Prioritize diracx/gubbins service logs - Add job summary with pod status - Add annotations for errors found in logs - Include infrastructure pods in separate group - Always run debugging step even on failure
1 parent 6525e65 commit d691175

2 files changed

Lines changed: 124 additions & 17 deletions

File tree

‎.github/scripts/collect-k8s-debug-info.sh‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
#!/usr/bin/env bash
2+
# Collect Kubernetes debug information with GitHub Actions formatting
3+
# Usage: collect-k8s-debug-info.sh <demo_dir> [service_prefix]
4+
# demo_dir: Path to demo directory containing kube.conf
5+
# service_prefix: Pod name prefix for main services (default: diracx)
6+
7+
set -euo pipefail
8+
9+
DEMO_DIR="${1:?Usage: $0 <demo_dir> [service_prefix]}"
10+
SERVICE_PREFIX="${2:-diracx}"
11+
12+
export KUBECONFIG="${DEMO_DIR}/kube.conf"
13+
export PATH="${DEMO_DIR}:$PATH"
14+
15+
INFRA_PODS="(dex|minio|mysql|rabbitmq|opensearch)"
16+
17+
# Function to output all info for a single pod
18+
output_pod_info() {
19+
local pod_name="$1"
20+
local tail_arg=("${@:2}")
21+
22+
echo "=== Pod Description ==="
23+
kubectl describe pod/"${pod_name}" 2>&1 || true
24+
25+
# Init containers
26+
init_containers=$(kubectl get pods "$pod_name" -o jsonpath='{.spec.initContainers[*].name}' 2>/dev/null || true)
27+
if [ -n "$init_containers" ]; then
28+
echo ""
29+
echo "=== Init Container Logs ==="
30+
for container_name in $init_containers; do
31+
echo "--- $container_name ---"
32+
kubectl logs "${pod_name}" -c "${container_name}" "${tail_arg[@]}" 2>&1 || true
33+
done
34+
fi
35+
36+
# Main containers
37+
echo ""
38+
echo "=== Container Logs ==="
39+
for container_name in $(kubectl get pods "$pod_name" -o jsonpath='{.spec.containers[*].name}'); do
40+
echo "--- $container_name ---"
41+
kubectl logs "${pod_name}" -c "${container_name}" "${tail_arg[@]}" 2>&1 || true
42+
done
43+
}
44+
45+
# === JOB SUMMARY ===
46+
{
47+
echo "## 🔍 Integration Test Debug Summary"
48+
echo ""
49+
echo "### Pod Status"
50+
echo '```'
51+
kubectl get pods -o wide
52+
echo '```'
53+
echo ""
54+
} >> "${GITHUB_STEP_SUMMARY:-/dev/null}"
55+
56+
# === POD STATUS OVERVIEW ===
57+
echo "::group::📋 Pod Status Overview"
58+
kubectl get pods -o wide
59+
echo "::endgroup::"
60+
61+
# === MAIN SERVICE PODS (shown first) ===
62+
for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -E "^${SERVICE_PREFIX}-" || true); do
63+
echo "::group::🚀 ${pod_name}"
64+
output_pod_info "$pod_name"
65+
echo "::endgroup::"
66+
done
67+
68+
# === OTHER APPLICATION PODS ===
69+
for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE "$INFRA_PODS" | grep -vE "^${SERVICE_PREFIX}-" || true); do
70+
if [ -n "$pod_name" ]; then
71+
echo "::group::📦 ${pod_name}"
72+
output_pod_info "$pod_name"
73+
echo "::endgroup::"
74+
fi
75+
done
76+
77+
# === INFRASTRUCTURE PODS ===
78+
for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -E "$INFRA_PODS" || true); do
79+
echo "::group::🏗️ ${pod_name}"
80+
output_pod_info "$pod_name" --tail=100
81+
echo "::endgroup::"
82+
done
83+
84+
# === ERROR SUMMARY WITH ANNOTATIONS ===
85+
echo "::group::⚠️ Error Summary"
86+
error_count=0
87+
for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -E "^${SERVICE_PREFIX}-" || true); do
88+
for container_name in $(kubectl get pods "$pod_name" -o jsonpath='{.spec.containers[*].name}'); do
89+
errors=$(kubectl logs "${pod_name}" -c "${container_name}" 2>&1 | grep -iE "(ERROR|CRITICAL|Exception|Traceback)" | head -5 || true)
90+
if [ -n "$errors" ]; then
91+
echo "Errors in ${pod_name}/${container_name}:"
92+
echo "$errors"
93+
echo ""
94+
# Add annotation (limited to avoid exceeding quota)
95+
if [ $error_count -lt 8 ]; then
96+
first_error=$(echo "$errors" | head -1 | cut -c1-200)
97+
echo "::warning title=Error in ${pod_name}/${container_name}::${first_error}"
98+
((error_count++)) || true
99+
fi
100+
fi
101+
done
102+
done
103+
if [ $error_count -eq 0 ]; then
104+
echo "No errors found in ${SERVICE_PREFIX} service logs"
105+
fi
106+
echo "::endgroup::"
107+
108+
# === CHECK FOR DEMO SUCCESS ===
109+
if [ ! -f "${DEMO_DIR}/.success" ]; then
110+
echo "::error title=Demo Failed::Demo did not complete successfully"
111+
{
112+
echo ""
113+
echo "## ❌ Demo Failed"
114+
echo '```'
115+
cat "${DEMO_DIR}/.failed" 2>/dev/null || echo "No failure details available"
116+
echo '```'
117+
} >> "${GITHUB_STEP_SUMMARY:-/dev/null}"
118+
cat "${DEMO_DIR}/.failed" 2>/dev/null || true
119+
exit 1
120+
fi
121+
122+
echo "✅ Demo completed successfully"

‎.github/workflows/main.yml‎

Lines changed: 2 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -205,23 +205,8 @@ jobs:
205205
set -x
206206
../diracx-charts/run_demo.sh "${demo_args[@]}" "${demo_source_dirs[@]}"
207207
- name: Debugging information
208-
run: |
209-
DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
210-
export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf
211-
export PATH=${DIRACX_DEMO_DIR}:$PATH
212-
kubectl get pods
213-
for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
214-
echo "${pod_name}"
215-
kubectl describe pod/"${pod_name}" || true
216-
for container_name in $(kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do
217-
echo $pod_name $container_name
218-
kubectl logs "${pod_name}" -c "${container_name}" || true
219-
done
220-
done
221-
if [ ! -f "${DIRACX_DEMO_DIR}/.success" ]; then
222-
cat "${DIRACX_DEMO_DIR}/.failed"
223-
exit 1
224-
fi
208+
if: always()
209+
run: .github/scripts/collect-k8s-debug-info.sh "$PWD/../diracx-charts/.demo" "${{ matrix.extension == 'gubbins' && 'gubbins' || 'diracx' }}"
225210
- name: Run pytest
226211
run: |
227212
pixi run pytest-${{ matrix.extension }} --demo-dir="$PWD/../diracx-charts/" --cov-report=xml:coverage-pytest.xml --junitxml=report.xml

0 commit comments

Comments
 (0)