Merge remote-tracking branch 'origin/main' into fix-thought-signature-pruning

Author: its-amann

.github/workflows/analyze-releases-for-adk-docs-updates.yml

@@ -45,15 +45,15 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v6
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
 
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6
         with:
           python-version: '3.11'
 
       - name: Load adk-bot SSH Private Key
-        uses: webfactory/ssh-agent@v0.9.1
+        uses: webfactory/ssh-agent@a6f90b1f127823b31d4d4a8d96047790581349bd # v0.9.1
         with:
           ssh-private-key: ${{ secrets.ADK_BOT_SSH_PRIVATE_KEY }}
 
@@ -64,7 +64,7 @@ jobs:
 
       - name: Restore session DB from cache
         if: ${{ github.event.inputs.resume == 'true' }}
-        uses: actions/cache/restore@v4
+        uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: contributing/samples/adk_team/adk_documentation/adk_release_analyzer/sessions.db
           key: analyzer-session-db-${{ github.run_id }}-${{ github.run_attempt }}
@@ -102,7 +102,7 @@ jobs:
 
       - name: Save session DB to cache
         if: always()
-        uses: actions/cache/save@v4
+        uses: actions/cache/save@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
         with:
           path: contributing/samples/adk_team/adk_documentation/adk_release_analyzer/sessions.db
           key: analyzer-session-db-${{ github.run_id }}-${{ github.run_attempt }}

.github/workflows/release-finalize.yml

@@ -59,7 +59,7 @@ jobs:
             echo "manifest_file=.github/.release-please-manifest.json" >> $GITHUB_OUTPUT
           fi
 
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
         if: steps.check.outputs.is_release_pr == 'true'
         with:
           ref: ${{ github.event.pull_request.base.ref }}

.github/workflows/release-update-adk-web.yaml

@@ -36,7 +36,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v6
+      uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
       with:
         persist-credentials: false
 
@@ -78,7 +78,7 @@ jobs:
         echo "email=$(echo "$USER_JSON" | jq -r '.id')+$(echo "$USER_JSON" | jq -r '.login')@users.noreply.github.com" >> $GITHUB_OUTPUT
 
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@v6
+      uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c # v6
       with:
         token: ${{ secrets.RELEASE_PAT }}
         commit-message: "Update compiled adk web files from ${{ github.event.inputs.adk_web_repo }}@${{ github.event.inputs.adk_web_tag || 'latest' }}"

.pre-commit-config.yaml

@@ -72,3 +72,11 @@ repos:
         exclude: (?i)SKILL\.md$
         additional_dependencies:
           - mdformat-gfm
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.4.2
+    hooks:
+      - id: codespell
+        # Configuration (skip globs, ignored words) lives in the
+        # [tool.codespell] table in pyproject.toml.
+        additional_dependencies:
+          - tomli

contributing/samples/evaluation/README.md

@@ -0,0 +1,64 @@
+# ADK evaluation samples
+
+## Overview
+
+A family of single-concept samples that each show one way to evaluate the *same*
+shared home-automation agent with the `adk eval` CLI. Every sample points `adk eval` at one agent and differs only in its eval data and criteria, so you can
+compare evaluation techniques (deterministic reference matching, custom metrics,
+LLM-as-a-judge, rubrics, and user simulation) side by side.
+
+## The shared agent
+
+`home_automation_agent/` is a small agent that controls smart-home devices and
+temperatures. Its five tools (`get_device_info`, `set_device_info`,
+`get_temperature`, `set_temperature`, `list_devices`) are deterministic, backed by
+in-memory state, so eval trajectories are reproducible. The module exposes
+`reset_data()`, which `adk eval` calls to reset that state between eval cases.
+
+Every sample evaluates this same agent. `adk eval` takes the agent path and the
+eval-set path as two separate arguments, so each sub-sample folder holds only
+eval data and its criteria config, never a copy of the agent code.
+
+## How evaluation runs
+
+`adk eval` runs in two phases: (1) live inference, where it actually runs the
+agent against each eval input to produce responses and tool calls, and then (2)
+scoring, where it compares that output against the case's criteria. Because phase 1
+runs the real agent, a model credential is required for every sample, even the
+deterministic ones. Provide a Gemini API key in `home_automation_agent/.env`,
+or configure Vertex.
+Samples that use an LLM judge or a user simulator make additional model calls, but
+they resolve through the same model registry and credentials.
+
+Because live responses vary from run to run, the deterministic, reference-based
+criteria use lenient response thresholds (e.g. `response_match_score` at
+`0.5`) so that harmless phrasing differences don't fail an otherwise-correct
+answer.
+
+## Samples
+
+| Sample                                            | Concept                                     | Criteria                                                                     |
+| ------------------------------------------------- | ------------------------------------------- | ---------------------------------------------------------------------------- |
+| [`basic_criteria`](./basic_criteria/)             | Deterministic, reference-based scoring      | `tool_trajectory_avg_score`, `response_match_score`                          |
+| [`test_file_vs_evalset`](./test_file_vs_evalset/) | `.test.json` vs `.evalset.json` conventions | `tool_trajectory_avg_score`, `response_match_score`                          |
+| [`custom_metric`](./custom_metric/)               | Write your own metric                       | `temperature_safety_score` (custom)                                          |
+| [`llm_judge_match`](./llm_judge_match/)           | LLM-judged semantic match                   | `final_response_match_v2`                                                    |
+| [`rubric_criteria`](./rubric_criteria/)           | LLM-judged quality via rubrics              | `rubric_based_final_response_quality_v1`, `rubric_based_tool_use_quality_v1` |
+| [`user_simulation`](./user_simulation/)           | Dynamically simulated user turns            | `hallucinations_v1`, `per_turn_user_simulator_quality_v1`                    |
+
+## Graph
+
+```mermaid
+graph TD
+    A[home_automation_agent] --> B(get_device_info)
+    A --> C(set_device_info)
+    A --> D(get_temperature)
+    A --> E(set_temperature)
+    A --> F(list_devices)
+```
+
+## Related Guides
+
+- Evaluation overview: https://adk.dev/evaluate/
+- Evaluation criteria reference: https://adk.dev/evaluate/criteria/
+- User simulation guide: https://adk.dev/evaluate/user-sim/

contributing/samples/evaluation/basic_criteria/README.md

@@ -0,0 +1,78 @@
+# Basic evaluation criteria
+
+## Overview
+
+Evaluates the shared home-automation agent with the two deterministic,
+reference-based criteria:
+
+- `tool_trajectory_avg_score`: does the agent call the right tools with the
+  right args? Each expected tool call (name + args) is compared against what the
+  agent actually did.
+- `response_match_score`: ROUGE-1 word overlap between the agent's final
+  response and a reference answer.
+
+Both criteria are computed locally with no judge model, so this sample needs only
+a model credential for the agent's own inference (a Gemini API key or Vertex).
+
+## Sample Inputs
+
+The eval set (`home_automation.evalset.json`) contains two single-turn cases:
+
+- `Turn off device_2.`
+- `What is the temperature in the Living Room?`
+
+## How To
+
+Run the sample from the workspace root:
+
+```bash
+adk eval contributing/samples/evaluation/home_automation_agent \
+    contributing/samples/evaluation/basic_criteria/home_automation.evalset.json \
+    --config_file_path contributing/samples/evaluation/basic_criteria/eval_config.json \
+    --print_detailed_results
+```
+
+`adk eval` takes the agent folder and the eval-set file as two separate
+arguments, so this folder holds only eval data (`home_automation.evalset.json`),
+the criteria config (`eval_config.json`), and this README, with no agent code.
+
+### `match_type` for tool trajectory
+
+`tool_trajectory_avg_score` has a `match_type` (set to `EXACT` here in
+`eval_config.json`) that controls how the expected and actual tool calls are
+compared:
+
+- `EXACT`: the actual tool calls must match the expected calls one-for-one, in
+  the same order, with identical args. Use this when the trajectory is fully
+  deterministic (as in this sample).
+- `IN_ORDER`: the expected calls must appear in the given order, but extra
+  actual calls in between are tolerated. Useful when the agent may take
+  additional, harmless steps.
+- `ANY_ORDER`: the expected calls must all appear, but order does not matter.
+  Useful when the agent may reorder independent tool calls.
+
+The `threshold` is `1.0`, so every expected call must match for the case to pass.
+
+### Why `response_match_score` uses a `0.6` threshold
+
+`adk eval` runs live inference, so the exact wording of the agent's final
+response varies from run to run (for example, "I have turned off device_2." vs
+"device_2 has been switched off."). `response_match_score` is a ROUGE-1 score,
+which measures word overlap rather than exact-string equality, so it tolerates
+this phrasing variation. The `0.6` threshold requires the response to share most
+of its wording with the reference while still allowing some rewording. Raise it
+toward `1.0` for stricter wording, lower it to tolerate more paraphrasing.
+
+### Expectations captured from a real run
+
+The expected `tool_uses` (tool names and args) in `home_automation.evalset.json`
+were captured from an actual `adk eval` run of the agent: run with
+`--print_detailed_results`, read the printed Actual-vs-Expected, then set the
+expected values to match what the agent really produced. The reference
+`final_response` for each case is an independently written natural answer (not a
+copy of the model output), which is exactly what ROUGE-1 is designed to tolerate.
+
+## Related Guides
+
+- Evaluation overview: https://adk.dev/evaluate/
+- Evaluation criteria reference: https://adk.dev/evaluate/criteria/

contributing/samples/evaluation/basic_criteria/eval_config.json

@@ -0,0 +1,9 @@
+{
+  "criteria": {
+    "tool_trajectory_avg_score": {
+      "threshold": 1.0,
+      "match_type": "EXACT"
+    },
+    "response_match_score": 0.6
+  }
+}

contributing/samples/evaluation/basic_criteria/home_automation.evalset.json

@@ -0,0 +1,62 @@
+{
+  "eval_set_id": "basic_criteria",
+  "name": "Basic deterministic criteria",
+  "description": "Single-turn cases scored by tool trajectory and ROUGE-1.",
+  "eval_cases": [
+    {
+      "eval_id": "turn_off_bedroom_device",
+      "conversation": [
+        {
+          "invocation_id": "basic-1",
+          "user_content": {
+            "parts": [{"text": "Turn off device_2."}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "device_2 is now turned off."}],
+            "role": "model"
+          },
+          "intermediate_data": {
+            "tool_uses": [
+              {"name": "set_device_info",
+               "args": {"device_id": "device_2", "status": "OFF"}}
+            ],
+            "intermediate_responses": []
+          }
+        }
+      ],
+      "session_input": {
+        "app_name": "home_automation_agent",
+        "user_id": "user",
+        "state": {}
+      }
+    },
+    {
+      "eval_id": "get_living_room_temperature",
+      "conversation": [
+        {
+          "invocation_id": "basic-2",
+          "user_content": {
+            "parts": [{"text": "What is the temperature in the Living Room?"}],
+            "role": "user"
+          },
+          "final_response": {
+            "parts": [{"text": "The Living Room is 22 degrees Celsius."}],
+            "role": "model"
+          },
+          "intermediate_data": {
+            "tool_uses": [
+              {"name": "get_temperature", "args": {"location": "Living Room"}}
+            ],
+            "intermediate_responses": []
+          }
+        }
+      ],
+      "session_input": {
+        "app_name": "home_automation_agent",
+        "user_id": "user",
+        "state": {}
+      }
+    }
+  ]
+}