Skip to content

Commit d77913f

Browse files
dumiansdumians
committed
chore: add CI configuration files and environment variable substitution script
1 parent d37a409 commit d77913f

6 files changed

Lines changed: 187 additions & 0 deletions

File tree

‎evals/ci_metadata.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
############################################################
16+
### CI Metadata (Repository Specific)
17+
### Note: These fields are used for version tracking in BQ
18+
### and are not part of the core Evalbench schema.
19+
############################################################
20+
21+
extension_id: oracledb
22+
release_version: ${RELEASE_VERSION}

‎evals/dataset.json‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
{
2+
"scenarios": [
3+
{
4+
"id": "oracledb-query-execution",
5+
"starting_prompt": "I want to run a query to get insights from my database.",
6+
"conversation_plan": "First, ask the agent to execute a simple SELECT query (e.g., SELECT * FROM users LIMIT 5) to verify connectivity and permissions. After the agent provides the results, ask it to execute a more complex query that involves a JOIN between two tables (e.g., SELECT orders.id, users.name FROM orders JOIN users ON orders.user_id = users.id LIMIT 5).",
7+
"expected_trajectory": [
8+
"execute_sql",
9+
"get_query_plan"
10+
],
11+
"env": {
12+
"ORACLE_CONNECTION_STRING": "${ORACLE_CONNECTION_STRING}",
13+
"ORACLE_PASSWORD": "${ORACLE_PASSWORD}",
14+
"ORACLE_USERNAME": "${ORACLE_USERNAME}",
15+
"ORACLE_WALLET": "${ORACLE_WALLET}",
16+
"ORACLE_USE_OCI": "${ORACLE_USE_OCI}"
17+
},
18+
"kind": "tools",
19+
"max_turns": 3
20+
},
21+
{
22+
"id": "oracledb-schema-tables-explore",
23+
"starting_prompt": "I want to understand the structure of my database.",
24+
"conversation_plan": "First, ask the agent to list tables and active sessions in the instance. After the agent provides the databases, ask it to list the tables specifically for that database.",
25+
"expected_trajectory": [
26+
"list_active_sessions",
27+
"list_tables"
28+
],
29+
"env": {
30+
"ORACLE_CONNECTION_STRING": "${ORACLE_CONNECTION_STRING}",
31+
"ORACLE_PASSWORD": "${ORACLE_PASSWORD}",
32+
"ORACLE_USERNAME": "${ORACLE_USERNAME}",
33+
"ORACLE_WALLET": "${ORACLE_WALLET}",
34+
"ORACLE_USE_OCI": "${ORACLE_USE_OCI}"
35+
},
36+
"kind": "tools",
37+
"max_turns": 3
38+
},
39+
{
40+
"id": "oracledb-performance-check",
41+
"starting_prompt": "Our database performance seems degraded.",
42+
"conversation_plan": "Start by asking the agent to check for any active queries that are running for a long time (e.g., more than 10 seconds). After the agent responds, follow up by asking if there are any highly fragmented tables that might be causing issues.",
43+
"expected_trajectory": [
44+
"list_tablespace_usage",
45+
"list_top_sql_by_resource"
46+
],
47+
"env": {
48+
"ORACLE_CONNECTION_STRING": "${ORACLE_CONNECTION_STRING}",
49+
"ORACLE_PASSWORD": "${ORACLE_PASSWORD}",
50+
"ORACLE_USERNAME": "${ORACLE_USERNAME}",
51+
"ORACLE_WALLET": "${ORACLE_WALLET}",
52+
"ORACLE_USE_OCI": "${ORACLE_USE_OCI}"
53+
},
54+
"kind": "tools",
55+
"max_turns": 3
56+
}
57+
]
58+
}

‎evals/gemini_2.5_pro_model.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
generator: gcp_vertex_gemini
16+
vertex_model: gemini-2.5-pro
17+
base_prompt: ""
18+
execs_per_minute: 5

‎evals/model_config.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
gemini_cli_version: "@google/gemini-cli@0.38.1"
16+
generator: gemini_cli
17+
env:
18+
GOOGLE_CLOUD_PROJECT: "${GOOGLE_CLOUD_PROJECT}"
19+
GOOGLE_CLOUD_LOCATION: "global"
20+
GOOGLE_GENAI_USE_VERTEXAI: "true"
21+
setup:
22+
extensions:
23+
# Points to the symlink created in cloudbuild.yaml to match the extension ID
24+
"/workspace/oracledb":
25+
settings:
26+
ORACLE_CONNECTION_STRING: "${ORACLE_CONNECTION_STRING}"
27+
ORACLE_PASSWORD: "${ORACLE_PASSWORD}"
28+
ORACLE_USERNAME: "${ORACLE_USERNAME}"
29+
ORACLE_WALLET: '${ORACLE_WALLET}'
30+
ORACLE_USE_OCI: "${ORACLE_USE_OCI}"

‎evals/run_config.yaml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Copyright 2025 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
dataset_config: /workspace/evals/dataset.json
16+
dataset_format: gemini-cli-format
17+
18+
orchestrator: geminicli
19+
model_config: /workspace/evals/model_config.yaml
20+
simulated_user_model_config: /workspace/evals/gemini_2.5_pro_model.yaml
21+
22+
scorers:
23+
# Qualitative (Judge-based)
24+
goal_completion:
25+
model_config: /workspace/evals/gemini_2.5_pro_model.yaml
26+
behavioral_metrics:
27+
model_config: /workspace/evals/gemini_2.5_pro_model.yaml
28+
skills_best_practices:
29+
model_config: /workspace/evals/gemini_2.5_pro_model.yaml
30+
skills_dir: /workspace/oracledb/skills
31+
32+
# Performance
33+
turn_count: {}
34+
end_to_end_latency: {}
35+
tool_call_latency: {}
36+
token_consumption: {}
37+
skills_trajectory: {}
38+
39+
reporting:
40+
bigquery:
41+
gcp_project_id: "${EVAL_REPORTING_PROJECT}"

‎evals/substitute_env.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
import os
2+
import re
3+
4+
def main():
5+
yaml_paths = ['/workspace/evals/model_config.yaml', '/workspace/evals/run_config.yaml', '/workspace/evals/dataset.json']
6+
for yaml_path in yaml_paths:
7+
if os.path.exists(yaml_path):
8+
with open(yaml_path, 'r') as f:
9+
content = f.read()
10+
content = re.sub(r'\${(\w+)}', lambda m: os.environ.get(m.group(1), m.group(0)), content)
11+
with open(yaml_path, 'w') as f:
12+
f.write(content)
13+
print(f"Successfully substituted environment variables in {yaml_path}")
14+
else:
15+
print(f"File not found: {yaml_path}")
16+
17+
if __name__ == '__main__':
18+
main()

0 commit comments

Comments
 (0)