ci: update evaluation configuration with skill tagging, model upgrade to 4-7, and restructured run settings

Author: omkargaikwad23

evals/claude_code_model.yaml

@@ -14,15 +14,18 @@
 
 claude_code_version: "@anthropic-ai/claude-code@2.1.85"
 generator: claude_code
-model: "claude-opus-4-6"
+model: "claude-opus-4-7"
 
 use_vertex: true
 vertex_project_id: "${GOOGLE_CLOUD_PROJECT}"
-vertex_region: "us-east5"
+vertex_region: "global"
 
 env:
-  CLOUD_ML_REGION: "us-east5"
+  # Global environment variables
+  CLOUD_ML_REGION: "global"
   GOOGLE_CLOUD_PROJECT: "${GOOGLE_CLOUD_PROJECT}"
+
+  # Cloud SQL PostgreSQL extension configuration
   CLOUD_SQL_POSTGRES_PROJECT: "${CLOUD_SQL_POSTGRES_PROJECT}"
   CLOUD_SQL_POSTGRES_INSTANCE: "${CLOUD_SQL_POSTGRES_INSTANCE}"
   CLOUD_SQL_POSTGRES_REGION: "${CLOUD_SQL_POSTGRES_REGION}"

evals/claude_run_config.yaml

@@ -12,13 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# Dataset Related Configs
 dataset_config: /workspace/evals/dataset.json
-dataset_format: gemini-cli-format
+dataset_format: agent-format
 
+# Orchestrator Configuration
 orchestrator: agent
 model_config: /workspace/evals/claude_code_model.yaml
 simulated_user_model_config: /workspace/evals/gemini_2.5_pro_model.yaml
 
+# Runner Related Configs
+runners:
+  agent_runners: 1
+
+# Scorer Related Configs
 scorers:
   # Qualitative (Judge-based)
   goal_completion:
@@ -36,6 +43,7 @@ scorers:
   token_consumption: {}
   skills_trajectory: {}
 
+# Reporting Related Configs
 reporting:
   bigquery:
     gcp_project_id: "${EVAL_REPORTING_PROJECT}"

evals/dataset.json

@@ -8,6 +8,9 @@
         "list_instances",
         "get_instance"
       ],
+      "expected_skills": [
+        "cloud-sql-postgres-admin"
+      ],
       "env": {
         "GOOGLE_CLOUD_PROJECT": "${GOOGLE_CLOUD_PROJECT}"
       },
@@ -22,6 +25,9 @@
         "list_schemas",
         "list_tables"
       ],
+      "expected_skills": [
+        "cloud-sql-postgres-data"
+      ],
       "env": {
         "GOOGLE_CLOUD_PROJECT": "${GOOGLE_CLOUD_PROJECT}"
       },
@@ -36,6 +42,9 @@
         "list_active_queries",
         "list_locks"
       ],
+      "expected_skills": [
+        "cloud-sql-postgres-monitor"
+      ],
       "env": {
         "GOOGLE_CLOUD_PROJECT": "${GOOGLE_CLOUD_PROJECT}"
       },
@@ -50,6 +59,9 @@
         "get_system_metrics",
         "list_database_stats"
       ],
+      "expected_skills": [
+        "cloud-sql-postgres-monitor"
+      ],
       "env": {
         "GOOGLE_CLOUD_PROJECT": "${GOOGLE_CLOUD_PROJECT}"
       },