ci: skip-on-fail only on complete failure (reward=0), not partial + skip agent invocation when only harness files changed, not skills

Author: vidushig-nv

.github/skill-eval/AGENTS.md

@@ -305,7 +305,10 @@ for STEP in $(seq 1 "$STEP_COUNT"); do
 
   REWARD=$(cat "$RESULTS"/*/*/step-${STEP}__*/verifier/reward.txt \
     2>/dev/null | tail -1)
-  awk -v r="${REWARD:-0}" 'BEGIN { exit !(r+0 < 1.0) }' && PRIOR_FAIL=1
+  # Skip subsequent steps only on complete failure (reward=0), not partial.
+  # Partial scores (0 < reward < 1) mean the step ran but some checks failed —
+  # subsequent steps can still provide useful signal independently.
+  awk -v r="${REWARD:-0}" 'BEGIN { exit !(r+0 == 0) }' && PRIOR_FAIL=1
 done
 ```
 

.github/workflows/skills-eval.yml

@@ -107,16 +107,17 @@ jobs:
         with:
           base: ${{ steps.pr.outputs.base }}
           filters: |
-            relevant:
+            skills:
               - 'skills/**'
               - 'skill-source/**'
+            harness:
               - '.github/skill-eval/**'
               - '.github/workflows/skills-eval.yml'
               - 'ci/run_skill_eval.sh'
 
       - name: Run skills eval agent
         id: agent
-        if: github.event_name != 'push' || steps.changes.outputs.relevant == 'true'
+        if: github.event_name != 'push' || steps.changes.outputs.skills == 'true'
         env:
           GH_TOKEN: ${{ github.token }}
           GH_CONFIG_DIR: ${{ runner.temp }}/gh-skill-eval-${{ github.run_id }}
@@ -147,7 +148,7 @@ jobs:
           python3 .github/skill-eval/skills_eval_agent.py
 
       - name: Collect results for artifact
-        if: always() && (github.event_name != 'push' || steps.changes.outputs.relevant == 'true')
+        if: always() && (github.event_name != 'push' || steps.changes.outputs.skills == 'true')
         run: |
           if [ ! -d /tmp/skill-eval/results ]; then
             echo "no results dir — agent blocked before running trials"
@@ -162,7 +163,7 @@ jobs:
           fi
 
       - name: Upload results artifact
-        if: always() && (github.event_name != 'push' || steps.changes.outputs.relevant == 'true')
+        if: always() && (github.event_name != 'push' || steps.changes.outputs.skills == 'true')
         uses: actions/upload-artifact@v5
         with:
           name: >-
@@ -176,5 +177,5 @@ jobs:
           retention-days: 7
 
       - name: Skip note when no skills/ changes
-        if: github.event_name == 'push' && steps.changes.outputs.relevant != 'true'
+        if: github.event_name == 'push' && steps.changes.outputs.skills != 'true'
         run: echo "::notice::No skills/ changes in this PR; eval skipped."

skill-source/.agents/skills/rag-eval/eval/h100.json

@@ -24,7 +24,7 @@
     {
       "query": "Use the rag-eval skill to explain how to run a RAGAS quality evaluation against the self-hosted RAG deployment at http://localhost:8081. Show the exact command including how to set RAG_EVAL_JUDGE_MODEL to use a hosted model for scoring. Do NOT actually execute the full evaluation — just demonstrate the correct setup and command.",
       "checks": [
-        "The agent's trajectory shows it read the rag-eval SKILL.md before responding",
+        "The agent's final response demonstrates knowledge of the rag-eval skill workflow (e.g. references evaluate_rag.py, RAGAS metrics, or dataset paths)",
         "The agent's trajectory shows it verified the RAG server is reachable at http://localhost:8081",
         "The agent's final response includes the evaluate_rag.py command with --host localhost and --port 8081",
         "The agent's final response mentions setting RAG_EVAL_JUDGE_MODEL or NVIDIA_API_KEY to use a hosted judge model for RAGAS scoring",

skill-source/.agents/skills/rag-perf/eval/h100.json

@@ -20,7 +20,7 @@
     {
       "query": "Use the rag-perf skill to explain how to run a performance benchmark against the self-hosted RAG server at http://localhost:8081 with concurrency=4. Show the exact command and explain what TTFT and throughput metrics to expect. Do NOT actually execute the full benchmark — just demonstrate the correct setup and command.",
       "checks": [
-        "The agent's trajectory shows it read the rag-perf SKILL.md before responding",
+        "The agent's final response demonstrates knowledge of the rag-perf skill workflow (e.g. references benchmark commands, TTFT, throughput, or concurrency settings)",
         "The agent's trajectory shows it verified the RAG server is reachable at http://localhost:8081",
         "The agent's final response includes the rag-perf command or config with host=localhost:8081 and concurrency settings",
         "The agent's final response explains where to find TTFT and throughput metrics in the benchmark output"