chore: add CI configuration files and environment variable substitution script

evals/ci_metadata.yaml

@@ -0,0 +1,22 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+############################################################
+### CI Metadata (Repository Specific)
+### Note: These fields are used for version tracking in BQ
+### and are not part of the core Evalbench schema.
+############################################################
+
+extension_id: oracledb
+release_version: ${RELEASE_VERSION}

evals/dataset.json

@@ -0,0 +1,58 @@
+{
+  "scenarios": [
+    {
+      "id": "oracledb-query-execution",
+      "starting_prompt": "I want to run a query to get insights from my database.",
+      "conversation_plan": "First, ask the agent to execute a simple SELECT query (e.g., SELECT * FROM users LIMIT 5) to verify connectivity and permissions. After the agent provides the results, ask it to execute a more complex query that involves a JOIN between two tables (e.g., SELECT orders.id, users.name FROM orders JOIN users ON orders.user_id = users.id LIMIT 5).",
+      "expected_trajectory": [
+        "execute_sql",
+        "get_query_plan"
+      ],
+      "env": {
+       "ORACLE_CONNECTION_STRING": "${ORACLE_CONNECTION_STRING}",
+        "ORACLE_PASSWORD": "${ORACLE_PASSWORD}",
+        "ORACLE_USERNAME": "${ORACLE_USERNAME}",
+        "ORACLE_WALLET": "${ORACLE_WALLET}",
+        "ORACLE_USE_OCI": "${ORACLE_USE_OCI}"
+      },
+      "kind": "tools",
+      "max_turns": 3
+    },
+    {
+      "id": "oracledb-schema-tables-explore",
+      "starting_prompt": "I want to understand the structure of my database.",
+      "conversation_plan": "First, ask the agent to list tables and active sessions in the instance. After the agent provides the databases, ask it to list the tables specifically for that database.",
+      "expected_trajectory": [
+        "list_active_sessions",
+        "list_tables"
+      ],
+      "env": {
+        "ORACLE_CONNECTION_STRING": "${ORACLE_CONNECTION_STRING}",
+        "ORACLE_PASSWORD": "${ORACLE_PASSWORD}",
+        "ORACLE_USERNAME": "${ORACLE_USERNAME}",
+        "ORACLE_WALLET": "${ORACLE_WALLET}",
+        "ORACLE_USE_OCI": "${ORACLE_USE_OCI}"
+      },
+      "kind": "tools",
+      "max_turns": 3
+    },
+    {
+      "id": "oracledb-performance-check",
+      "starting_prompt": "Our database performance seems degraded.",
+      "conversation_plan": "Start by asking the agent to check for any active queries that are running for a long time (e.g., more than 10 seconds). After the agent responds, follow up by asking if there are any highly fragmented tables that might be causing issues.",
+      "expected_trajectory": [
+        "list_tablespace_usage",
+        "list_top_sql_by_resource"
+      ],
+      "env": {
+        "ORACLE_CONNECTION_STRING": "${ORACLE_CONNECTION_STRING}",
+        "ORACLE_PASSWORD": "${ORACLE_PASSWORD}",
+        "ORACLE_USERNAME": "${ORACLE_USERNAME}",
+        "ORACLE_WALLET": "${ORACLE_WALLET}",
+        "ORACLE_USE_OCI": "${ORACLE_USE_OCI}"
+      },
+      "kind": "tools",
+      "max_turns": 3
+    }
+  ]
+}

evals/gemini_2.5_pro_model.yaml

@@ -0,0 +1,18 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+generator: gcp_vertex_gemini
+vertex_model: gemini-2.5-pro
+base_prompt: ""
+execs_per_minute: 5

evals/model_config.yaml

@@ -0,0 +1,30 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+gemini_cli_version: "@google/gemini-cli@0.38.1"
+generator: gemini_cli
+env:
+  GOOGLE_CLOUD_PROJECT: "${GOOGLE_CLOUD_PROJECT}"
+  GOOGLE_CLOUD_LOCATION: "global"
+  GOOGLE_GENAI_USE_VERTEXAI: "true"
+setup:
+  extensions:
+    # Points to the symlink created in cloudbuild.yaml to match the extension ID
+    "/workspace/oracledb":
+      settings:
+        ORACLE_CONNECTION_STRING: "${ORACLE_CONNECTION_STRING}"
+        ORACLE_PASSWORD: "${ORACLE_PASSWORD}"
+        ORACLE_USERNAME: "${ORACLE_USERNAME}"
+        ORACLE_WALLET: '${ORACLE_WALLET}'
+        ORACLE_USE_OCI: "${ORACLE_USE_OCI}"

evals/run_config.yaml

@@ -0,0 +1,41 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+dataset_config: /workspace/evals/dataset.json
+dataset_format: gemini-cli-format
+
+orchestrator: geminicli
+model_config: /workspace/evals/model_config.yaml
+simulated_user_model_config: /workspace/evals/gemini_2.5_pro_model.yaml
+
+scorers:
+  # Qualitative (Judge-based)
+  goal_completion:
+    model_config: /workspace/evals/gemini_2.5_pro_model.yaml
+  behavioral_metrics:
+    model_config: /workspace/evals/gemini_2.5_pro_model.yaml
+  skills_best_practices:
+    model_config: /workspace/evals/gemini_2.5_pro_model.yaml
+    skills_dir: /workspace/oracledb/skills
+
+  # Performance
+  turn_count: {}
+  end_to_end_latency: {}
+  tool_call_latency: {}
+  token_consumption: {}
+  skills_trajectory: {}
+
+reporting:
+  bigquery:
+    gcp_project_id: "${EVAL_REPORTING_PROJECT}"

evals/substitute_env.py

@@ -0,0 +1,18 @@
+import os
+import re
+
+def main():
+    yaml_paths = ['/workspace/evals/model_config.yaml', '/workspace/evals/run_config.yaml', '/workspace/evals/dataset.json']
+    for yaml_path in yaml_paths:
+        if os.path.exists(yaml_path):
+            with open(yaml_path, 'r') as f:
+                content = f.read()
+            content = re.sub(r'\${(\w+)}', lambda m: os.environ.get(m.group(1), m.group(0)), content)
+            with open(yaml_path, 'w') as f:
+                f.write(content)
+            print(f"Successfully substituted environment variables in {yaml_path}")
+        else:
+            print(f"File not found: {yaml_path}")
+
+if __name__ == '__main__':
+    main()