From dcf548a0d4f4a29dfc55c10d538e0c835a90a8cc Mon Sep 17 00:00:00 2001 From: dtehan-td Date: Tue, 23 Jun 2026 16:30:39 -0700 Subject: [PATCH 1/6] chore: remove temporary seed and DDL extraction files These files (seed_tdml_summaries.py and extracted DDL scripts) were created during development for testing database connectivity and seeding teradataml summaries. They are no longer needed and should not be version-controlled. --- ddls_extracted/evals_employees_DDL.sql | 25 ---- ddls_extracted/evals_orders_DDL.sql | 25 ---- scripts/seed_tdml_summaries.py | 198 ------------------------- 3 files changed, 248 deletions(-) delete mode 100644 ddls_extracted/evals_employees_DDL.sql delete mode 100644 ddls_extracted/evals_orders_DDL.sql delete mode 100644 scripts/seed_tdml_summaries.py diff --git a/ddls_extracted/evals_employees_DDL.sql b/ddls_extracted/evals_employees_DDL.sql deleted file mode 100644 index 935ffe2..0000000 --- a/ddls_extracted/evals_employees_DDL.sql +++ /dev/null @@ -1,25 +0,0 @@ -/* - * File: evals_employees_DDL.sql - * Generated: 2026-06-23 14:16:55 - * Type: TABLE - * Database: demo_user - * Object: evals_employees - * Size: 572 characters - */ - -CREATE SET TABLE demo_user.evals_employees ,FALLBACK , - NO BEFORE JOURNAL, - NO AFTER JOURNAL, - CHECKSUM = DEFAULT, - DEFAULT MERGEBLOCKRATIO, - MAP = TD_MAP1 - ( - employee_id INTEGER NOT NULL, - name VARCHAR(100) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL, - department VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL, - salary DECIMAL(10,2), - region VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL, - hire_date DATE FORMAT 'YY/MM/DD' NOT NULL, - manager_id INTEGER, -PRIMARY KEY ( employee_id )) -; \ No newline at end of file diff --git a/ddls_extracted/evals_orders_DDL.sql b/ddls_extracted/evals_orders_DDL.sql deleted file mode 100644 index 8c90da9..0000000 --- a/ddls_extracted/evals_orders_DDL.sql +++ /dev/null @@ -1,25 +0,0 @@ -/* - * File: evals_orders_DDL.sql - * Generated: 2026-06-23 14:16:30 - * Type: TABLE - * Database: demo_user - * Object: evals_orders - * Size: 563 characters - */ - -CREATE SET TABLE demo_user.evals_orders ,FALLBACK , - NO BEFORE JOURNAL, - NO AFTER JOURNAL, - CHECKSUM = DEFAULT, - DEFAULT MERGEBLOCKRATIO, - MAP = TD_MAP1 - ( - order_id INTEGER NOT NULL, - customer_name VARCHAR(100) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL, - order_date DATE FORMAT 'YY/MM/DD' NOT NULL, - ship_date DATE FORMAT 'YY/MM/DD', - amount DECIMAL(10,2) NOT NULL, - product_category VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL, - quantity INTEGER NOT NULL, -PRIMARY KEY ( order_id )) -; \ No newline at end of file diff --git a/scripts/seed_tdml_summaries.py b/scripts/seed_tdml_summaries.py deleted file mode 100644 index 20bc392..0000000 --- a/scripts/seed_tdml_summaries.py +++ /dev/null @@ -1,198 +0,0 @@ -""" -Seed script: connects to Teradata, extracts one-line summaries from teradataml -__init__.__doc__ for all TD_ANALYTIC_FUNCS, then prints the new dict[str, str] -block ready to paste into constants.py. - -Requires DATABASE_URI env var: - export DATABASE_URI="teradata://user:pass@host:1025/db" - uv run python scripts/seed_tdml_summaries.py -""" - -import os -import re -import textwrap -import warnings - -warnings.filterwarnings("ignore") - -import teradataml as tdml # noqa: E402 - -# Connect so that teradataml populates __init__.__doc__ on each class -_uri = os.environ.get("DATABASE_URI", "") -if _uri: - _m = re.match(r"teradata://([^:]+):([^@]+)@([^:]+):(\d+)/(.+)", _uri) - if _m: - tdml.create_context( - host=_m.group(3), - username=_m.group(1), - password=_m.group(2), - database=_m.group(5), - ) - else: - raise ValueError(f"Cannot parse DATABASE_URI: {_uri}") -else: - raise EnvironmentError("DATABASE_URI not set — docstrings require a live connection") - -FUNCS = [ - "ANOVA", - "Attribution", - "Antiselect", - "Apriori", - "BincodeFit", - "BincodeTransform", - "CFilter", - "CategoricalSummary", - "ChiSq", - "ClassificationEvaluator", - "ColumnSummary", - "ColumnTransformer", - "ConvertTo", - "DecisionForest", - "FTest", - "FillRowId", - "Fit", - "GetFutileColumns", - "GetRowsWithMissingValues", - "GetRowsWithoutMissingValues", - "GLM", - "GLMPerSegment", - "Histogram", - "KMeans", - "KMeansPredict", - "KNN", - "MovingAverage", - "NERExtractor", - "NGramSplitter", - "NaiveBayesTextClassifierPredict", - "NaiveBayesTextClassifierTrainer", - "NonLinearCombineFit", - "NonLinearCombineTransform", - "NumApply", - "NPath", - "OneClassSVM", - "OneClassSVMPredict", - "OneHotEncodingFit", - "OneHotEncodingTransform", - "OrdinalEncodingFit", - "OrdinalEncodingTransform", - "OutlierFilterFit", - "OutlierFilterTransform", - "Pack", - "PolynomialFeaturesFit", - "PolynomialFeaturesTransform", - "Pivoting", - "QQNorm", - "ROC", - "RandomProjectionFit", - "RandomProjectionMinComponents", - "RandomProjectionTransform", - "RegressionEvaluator", - "RoundColumns", - "RowNormalizeFit", - "RowNormalizeTransform", - "SMOTE", - "SVM", - "SVMPredict", - "ScaleFit", - "ScaleTransform", - "Sessionize", - "SentimentExtractor", - "Shap", - "Silhouette", - "SimpleImputeFit", - "SimpleImputeTransform", - "StrApply", - "StringSimilarity", - "TDDecisionForestPredict", - "TDGLMPredict", - "TDNaiveBayesPredict", - "TFIDF", - "TargetEncodingFit", - "TargetEncodingTransform", - "TextMorph", - "TextParser", - "TrainTestSplit", - "Transform", - "UnivariateStatistics", - "Unpack", - "Unpivoting", - "VectorDistance", - "WhichMax", - "WhichMin", - "WordEmbeddings", - "XGBoost", - "XGBoostPredict", - "ZTest", -] - - -def extract_summary(func_name: str) -> str: - """Pull the first meaningful sentence from the teradataml __init__ docstring.""" - func_obj = getattr(tdml, func_name, None) - if func_obj is None: - return f"Teradata ML analytic function {func_name}." - - raw = getattr(func_obj.__init__, "__doc__", None) or "" - # Dedent and strip leading blank lines - raw = textwrap.dedent(raw).strip() - - # The teradataml pattern is: - # DESCRIPTION: - # - # - # PARAMETERS: - # Try to grab the DESCRIPTION block first. - desc_match = re.search(r"DESCRIPTION\s*:\s*\n(.*?)(?:\n\s*\n|\n\s*PARAMETERS\s*:)", raw, re.DOTALL) - if desc_match: - block = desc_match.group(1) - else: - # Fallback: take the first non-empty paragraph - block = raw.split("\n\n")[0] - - # Collapse internal whitespace / newlines into a single line - block = re.sub(r"\s+", " ", block).strip() - - # Replace teradataml-specific terminology - block = block.replace("teradataml DataFrame", "table name") - block = block.replace("DataFrame", "table name") - - # Truncate at the first sentence boundary (period followed by space or end) - # Keep the trailing period. - sent_match = re.search(r"^(.*?\.)\s", block) - if sent_match: - summary = sent_match.group(1) - else: - # No sentence boundary — use the whole block but cap length - summary = block[:200].rstrip() - if not summary.endswith("."): - summary += "." - - return summary - - -def main(): - results: list[tuple[str, str]] = [] - missing: list[str] = [] - - for name in FUNCS: - summary = extract_summary(name) - results.append((name, summary)) - if "analytic function" in summary and name in summary: - missing.append(name) - - # Print the dict literal ready to paste into constants.py - print("TD_ANALYTIC_FUNCS = {") - for name, summary in results: - # Escape any quotes inside the summary - safe = summary.replace('"', '\\"') - print(f' "{name}": "{safe}",') - print("}") - - if missing: - print(f"\n# WARNING: {len(missing)} functions had no extractable docstring — fallback used:") - for m in missing: - print(f"# {m}") - - -if __name__ == "__main__": - main() From 99c17bf75cfab9594c5070905ebd03ea39f4f49f Mon Sep 17 00:00:00 2001 From: dtehan-td Date: Tue, 23 Jun 2026 16:34:24 -0700 Subject: [PATCH 2/6] chore: upgrade langsmith to >=0.8.18 Add explicit langsmith dependency with minimum version 0.8.18. Updates lock file to 0.9.1, the latest compatible version. --- pyproject.toml | 1 + uv.lock | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ca299dd..930eea4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "PyYAML>=6.0.0", "sqlalchemy>=2.0.0,<3.0.0", "starlette>=1.3.1", + "langsmith>=0.8.18", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index bebc348..009378a 100644 --- a/uv.lock +++ b/uv.lock @@ -2046,23 +2046,27 @@ wheels = [ [[package]] name = "langsmith" -version = "0.8.16" +version = "0.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "anyio" }, + { name = "distro" }, { name = "httpx" }, { name = "orjson", marker = "platform_python_implementation != 'PyPy'" }, { name = "packaging" }, { name = "pydantic" }, { name = "requests" }, { name = "requests-toolbelt" }, + { name = "sniffio" }, + { name = "typing-extensions" }, { name = "uuid-utils" }, { name = "websockets" }, { name = "xxhash" }, { name = "zstandard" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6f/19/1ed2af9c6d5d7a148e6b3e809b0af8ce8848e1f66a0726c8223d30e5292b/langsmith-0.8.16.tar.gz", hash = "sha256:8c943f0c9185fe2a9637b5b442828b7efd823b1de28d50d14c136c79660f909b", size = 4513275 } +sdist = { url = "https://files.pythonhosted.org/packages/5b/26/b72987d947278f63ec1e85f01ce85ca7ab2621c7efc0845d4a3a8e5d5dfb/langsmith-0.9.1.tar.gz", hash = "sha256:e5eb905224d156bcece4985285c55b51fffcb06c9353b2c4adb42e1c48b0d05d", size = 4557557 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/13/8186a9867c67f3fef9958a1d60b45f46c1a9b5d28f67d8fd136f28ceab3f/langsmith-0.8.16-py3-none-any.whl", hash = "sha256:081e57c0175d142192683288740a796eb0eb32d9e703b4bf9133678ceefe3286", size = 500303 }, + { url = "https://files.pythonhosted.org/packages/63/42/13c67eb24cddb368df2c8af13e319c86b1d270c90f5b8c9e8baed3593e04/langsmith-0.9.1-py3-none-any.whl", hash = "sha256:1160bf667af63d9bc081821f1df351cb84f7875740858f2a97ffef62b21800a9", size = 578856 }, ] [[package]] @@ -4458,6 +4462,7 @@ version = "0.2.4" source = { editable = "." } dependencies = [ { name = "fastmcp" }, + { name = "langsmith" }, { name = "mcp", extra = ["cli"] }, { name = "pydantic" }, { name = "python-dotenv" }, @@ -4492,6 +4497,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "fastmcp", specifier = ">=3.2.0,<3.4.2" }, + { name = "langsmith", specifier = ">=0.8.18" }, { name = "mcp", extras = ["cli"], specifier = ">=1.0.0,<2.0.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.5.0" }, { name = "pydantic", specifier = ">=2.8.0,<3.0.0" }, From e48b526afc5d3403e5c38466520ff650e9a6fbf2 Mon Sep 17 00:00:00 2001 From: dtehan-td Date: Tue, 23 Jun 2026 16:35:24 -0700 Subject: [PATCH 3/6] chore: upgrade pydantic-settings to >=2.14.2 Add explicit pydantic-settings dependency with minimum version 2.14.2. Updates lock file to 2.14.2. --- pyproject.toml | 1 + uv.lock | 8 +++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 930eea4..9a45208 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,6 +24,7 @@ classifiers = [ ] dependencies = [ "pydantic>=2.8.0,<3.0.0", + "pydantic-settings>=2.14.2", "fastmcp>=3.2.0,<3.4.2", "mcp[cli]>=1.0.0,<2.0.0", "teradatasqlalchemy>=20.0.0.0", diff --git a/uv.lock b/uv.lock index 009378a..867f9ec 100644 --- a/uv.lock +++ b/uv.lock @@ -3678,16 +3678,16 @@ wheels = [ [[package]] name = "pydantic-settings" -version = "2.14.1" +version = "2.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "python-dotenv" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551 } +sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700 } wheels = [ - { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964 }, + { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715 }, ] [[package]] @@ -4465,6 +4465,7 @@ dependencies = [ { name = "langsmith" }, { name = "mcp", extra = ["cli"] }, { name = "pydantic" }, + { name = "pydantic-settings" }, { name = "python-dotenv" }, { name = "pyyaml" }, { name = "sqlalchemy" }, @@ -4501,6 +4502,7 @@ requires-dist = [ { name = "mcp", extras = ["cli"], specifier = ">=1.0.0,<2.0.0" }, { name = "mypy", marker = "extra == 'dev'", specifier = ">=1.5.0" }, { name = "pydantic", specifier = ">=2.8.0,<3.0.0" }, + { name = "pydantic-settings", specifier = ">=2.14.2" }, { name = "python-dotenv", specifier = ">=1.0.0" }, { name = "pyyaml", specifier = ">=6.0.0" }, { name = "requests", marker = "extra == 'bar'", specifier = ">=2.25.0" }, From 6f11bfcb38890a10db29929f1e326e9f80110e97 Mon Sep 17 00:00:00 2001 From: dtehan-td Date: Tue, 23 Jun 2026 17:10:17 -0700 Subject: [PATCH 4/6] Improve error reporting in evals: unwrap ExceptionGroup to show actual error --- .../.deepeval/.deepeval_telemetry.txt | 4 + tests/eval_test/.env.example | 33 + tests/eval_test/AGENTS.md | 189 + tests/eval_test/README.md | 39 + tests/eval_test/agent/__init__.py | 0 tests/eval_test/agent/client.py | 372 ++ tests/eval_test/backup/README.md | 122 + tests/eval_test/backup/__init__.py | 1 + tests/eval_test/backup/audit_cases.py | 291 ++ tests/eval_test/backup/generate_cases.py | 175 + tests/eval_test/backup/inspect_tools.py | 92 + tests/eval_test/cases/base.json | 460 +++ tests/eval_test/cases/chat.json | 19 + tests/eval_test/cases/dba.json | 441 +++ tests/eval_test/cases/plot.json | 138 + tests/eval_test/cases/qlty.json | 183 + tests/eval_test/cases/sec.json | 231 ++ tests/eval_test/cases/tmpl.json | 9 + tests/eval_test/description_overrides.json | 4 + tests/eval_test/docs/cases.md | 97 + tests/eval_test/docs/setup.md | 94 + tests/eval_test/docs/structure.md | 35 + tests/eval_test/docs/workflow.md | 133 + tests/eval_test/judge/__init__.py | 0 tests/eval_test/judge/bedrock_llm.py | 57 + tests/eval_test/judge/checks.py | 114 + tests/eval_test/judge/metrics.py | 43 + tests/eval_test/judge/report.py | 561 +++ tests/eval_test/judge/suggest_overrides.py | 657 ++++ tests/eval_test/preflight.py | 74 + tests/eval_test/pyproject.toml | 31 + tests/eval_test/results/index.json | 46 + tests/eval_test/results/latest.json | 15 + tests/eval_test/results/latest_summary.json | 683 ++++ tests/eval_test/results/latest_summary.md | 867 +++++ .../manifest.json | 22 + .../summary.json | 3209 +++++++++++++++++ .../summary.md | 268 ++ .../manifest.json | 22 + .../summary.json | 683 ++++ .../summary.md | 867 +++++ .../manifest.json | 22 + .../summary.json | 683 ++++ .../summary.md | 867 +++++ tests/eval_test/run_evals.py | 136 + tests/eval_test/setup_test_data.py | 132 + tests/eval_test/suggest_overrides.py | 163 + tests/eval_test/teardown_test_data.py | 43 + tests/eval_test/tests/__init__.py | 0 tests/eval_test/tests/case_runner.py | 435 +++ tests/eval_test/tests/conftest.py | 107 + tests/eval_test/uv.lock | 2101 +++++++++++ 52 files changed, 16070 insertions(+) create mode 100644 tests/eval_test/.deepeval/.deepeval_telemetry.txt create mode 100644 tests/eval_test/.env.example create mode 100644 tests/eval_test/AGENTS.md create mode 100644 tests/eval_test/README.md create mode 100644 tests/eval_test/agent/__init__.py create mode 100644 tests/eval_test/agent/client.py create mode 100644 tests/eval_test/backup/README.md create mode 100644 tests/eval_test/backup/__init__.py create mode 100644 tests/eval_test/backup/audit_cases.py create mode 100644 tests/eval_test/backup/generate_cases.py create mode 100644 tests/eval_test/backup/inspect_tools.py create mode 100644 tests/eval_test/cases/base.json create mode 100644 tests/eval_test/cases/chat.json create mode 100644 tests/eval_test/cases/dba.json create mode 100644 tests/eval_test/cases/plot.json create mode 100644 tests/eval_test/cases/qlty.json create mode 100644 tests/eval_test/cases/sec.json create mode 100644 tests/eval_test/cases/tmpl.json create mode 100644 tests/eval_test/description_overrides.json create mode 100644 tests/eval_test/docs/cases.md create mode 100644 tests/eval_test/docs/setup.md create mode 100644 tests/eval_test/docs/structure.md create mode 100644 tests/eval_test/docs/workflow.md create mode 100644 tests/eval_test/judge/__init__.py create mode 100644 tests/eval_test/judge/bedrock_llm.py create mode 100644 tests/eval_test/judge/checks.py create mode 100644 tests/eval_test/judge/metrics.py create mode 100644 tests/eval_test/judge/report.py create mode 100644 tests/eval_test/judge/suggest_overrides.py create mode 100644 tests/eval_test/preflight.py create mode 100644 tests/eval_test/pyproject.toml create mode 100644 tests/eval_test/results/index.json create mode 100644 tests/eval_test/results/latest.json create mode 100644 tests/eval_test/results/latest_summary.json create mode 100644 tests/eval_test/results/latest_summary.md create mode 100644 tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/manifest.json create mode 100644 tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.json create mode 100644 tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.md create mode 100644 tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/manifest.json create mode 100644 tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.json create mode 100644 tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.md create mode 100644 tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/manifest.json create mode 100644 tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json create mode 100644 tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md create mode 100644 tests/eval_test/run_evals.py create mode 100644 tests/eval_test/setup_test_data.py create mode 100644 tests/eval_test/suggest_overrides.py create mode 100644 tests/eval_test/teardown_test_data.py create mode 100644 tests/eval_test/tests/__init__.py create mode 100644 tests/eval_test/tests/case_runner.py create mode 100644 tests/eval_test/tests/conftest.py create mode 100644 tests/eval_test/uv.lock diff --git a/tests/eval_test/.deepeval/.deepeval_telemetry.txt b/tests/eval_test/.deepeval/.deepeval_telemetry.txt new file mode 100644 index 0000000..3d80a6d --- /dev/null +++ b/tests/eval_test/.deepeval/.deepeval_telemetry.txt @@ -0,0 +1,4 @@ +DEEPEVAL_ID=66f13728-e025-470d-b8fc-582a61a9d898 +DEEPEVAL_STATUS=old +DEEPEVAL_LAST_FEATURE=evaluation +DEEPEVAL_EVALUATION_STATUS=old diff --git a/tests/eval_test/.env.example b/tests/eval_test/.env.example new file mode 100644 index 0000000..7a71e4e --- /dev/null +++ b/tests/eval_test/.env.example @@ -0,0 +1,33 @@ +# AWS credentials — boto3 uses the standard credential chain: +# env vars → ~/.aws/credentials → IAM instance profile +AWS_REGION=us-east-1 +AWS_ACCESS_KEY_ID=your-access-key-id +AWS_SECRET_ACCESS_KEY=your-secret-access-key +# AWS_SESSION_TOKEN=your-session-token # only needed for temporary credentials + +# Claude model ID on Bedrock — used for both agent and judge unless overridden +BEDROCK_MODEL_ID=anthropic.claude-3-5-sonnet-20241022-v2:0 + +# Override judge model separately (optional — defaults to BEDROCK_MODEL_ID) +# BEDROCK_JUDGE_MODEL_ID=anthropic.claude-3-5-sonnet-20241022-v2:0 + +# MCP server URL (must be running before evals start) +MCP_SERVER_URL=http://127.0.0.1:8001/mcp + +# Database where evals_employees and evals_orders are created (usually your ClearScape username) +EVALS_DATABASE=your_database_name + +# Teradata connection — used by setup_test_data.py and teardown_test_data.py only +TERADATA_HOST=your-clearscape-host.teradata.com +TERADATA_USER=your_username +TERADATA_PASSWORD=your_password + +# Max tool-use turns per eval case (single-turn cases) +AGENT_MAX_STEPS=5 + +# Max tool-use turns per conversation turn (multi-turn cases) +AGENT_MAX_STEPS_PER_TURN=3 + +# Tool descriptions seen by the agent during evals (default: live MCP server) +# USE_DESCRIPTION_OVERRIDES=1 +# DESCRIPTION_OVERRIDES_FILE=description_overrides.json \ No newline at end of file diff --git a/tests/eval_test/AGENTS.md b/tests/eval_test/AGENTS.md new file mode 100644 index 0000000..85b4481 --- /dev/null +++ b/tests/eval_test/AGENTS.md @@ -0,0 +1,189 @@ +# PROJECT KNOWLEDGE BASE + +**Updated:** 2026-06-23 + +## OVERVIEW + +**Project:** teradata-mcp-evals + +**Purpose:** Eval suite for the [Teradata MCP Server](https://github.com/Teradata/teradata-mcp-server) community edition. Primary goal is **MCP tool description quality** — tests whether an LLM agent selects the right tool and forms valid parameters from natural language. Failures on `ambiguous_selection` cases usually indicate overlapping or unclear tool descriptions in the MCP server. + +**Stack:** +- **Language:** Python ≥ 3.11 +- **Package manager / build tool:** `uv` (recommended) or standard `venv` + `pip` +- **Core dependencies:** + - `deepeval>=1.4.0` + - `mcp>=1.0.0` + - `boto3>=1.34.0` + - `python-dotenv>=1.0.0` + - `pytest>=8.0.0` + - `pytest-asyncio>=0.23.0` + - `teradatasql>=20.0.0` +- **Auxiliary tools:** `uv`, `dotenv`, deepeval CLI, `teradatasql` driver, AWS Bedrock (Claude) for agent + judge LLM calls + +**Case types:** + +| Type | Purpose | +|------|---------| +| `happy_path` | Unambiguous prompt → correct tool + params | +| `ambiguous_selection` | Two tools could apply → tests description distinctness | +| `missing_parameter` | Vague prompt → agent must ask, not hallucinate | +| `multi_tool` | Chained tool calls in order | +| `multi_turn` (optional) | 2–7 turns: clarify first, then correct tool after user supplies info | + +**Module priority:** P0 `base` · P1 `dba`, `sec` · P2 `qlty` · maintained `chat`, `plot`, `tmpl` + +## STRUCTURE + +``` +agent/ + client.py # MCP agent; optional description_overrides (opt-in) +judge/ + bedrock_llm.py # DeepEvalBaseLLM wrapper for Bedrock Converse API + checks.py # Deterministic structural checks (fail before LLM judge) + metrics.py # ToolCorrectnessMetric + Clarification GEval + report.py # Eval summaries → results/latest_summary.* + suggest_overrides.py # LLM draft overrides from failed ambiguous_selection cases +cases/ + .json # Eval case definitions per MCP module prefix +tests/ + conftest.py # Fixtures, {EVALS_DATABASE} substitution, assert_eval_case() + case_runner.py # Single/multi-turn execution and per-turn scoring + test_.py # One pytest file per module (all use assert_eval_case) + test_checks.py # Unit tests for judge/checks.py + test_multi_turn.py # Unit tests for multi-turn schema validation +backup/ # Optional bootstrap/audit scripts — see backup/README.md +run_evals.py # CLI entry point (deepeval + pytest) +suggest_overrides.py # CLI: draft description_overrides from eval failures +setup_test_data.py # Create evals_employees / evals_orders in EVALS_DATABASE +preflight.py # Verify eval tables exist before live eval runs +teardown_test_data.py +docs/ # Extended user documentation +README.md # Quick start and doc index +.env.example # Required environment variables +pyproject.toml # Project metadata, Ruff config, pytest paths +``` + +## COMMANDS + +| Action | Command | +|--------|---------| +| **Install dependencies** (recommended) | `curl -LsSf https://astral.sh/uv/install.sh \| sh && uv venv && uv sync` | +| **Install dependencies** (standard) | `python3 -m venv .venv && source .venv/bin/activate && pip install -e .` | +| **Run test data setup** | `python setup_test_data.py` (or `uv run python setup_test_data.py`) | +| **Verify eval tables** | `python preflight.py` (also runs automatically via `run_evals.py`) | +| **Generate happy-path cases** | `uv run python backup/generate_cases.py` | +| **Regenerate after description change** | `uv run python backup/generate_cases.py --module base --overwrite` | +| **Audit ambiguous pair coverage** | `uv run python backup/audit_cases.py --strict` | +| **Audit against live MCP tools** | `uv run python backup/audit_cases.py --live-mcp --strict` | +| **List eval runs** | `uv run python run_evals.py --list-runs` | +| **Suggest description overrides** | `uv run python suggest_overrides.py` (after baseline eval) | +| **Apply description overrides** | `uv run python suggest_overrides.py --apply` (after reviewing draft) | +| **Test description overrides** | `uv run python run_evals.py --with-description-overrides` | +| **Unit tests (local)** | `uv run pytest tests/test_checks.py tests/test_multi_turn.py tests/test_report.py tests/test_suggest_overrides.py tests/test_description_overrides.py -v` | +| **Run full eval suite** | `python run_evals.py` (or `uv run python run_evals.py`) | +| **Run one module** | `python run_evals.py --module base` | +| **Filter by case type** | `python run_evals.py --module base --type ambiguous_selection` | +| **Run multi-turn cases only** | `python run_evals.py --module base --type multi_turn` | +| **Verbose output** | `python run_evals.py --verbose` | +| **Unit tests (no MCP/Bedrock)** | `uv run pytest tests/test_checks.py tests/test_multi_turn.py tests/test_report.py tests/test_suggest_overrides.py tests/test_description_overrides.py` | +| **Run tests directly (pytest)** | `pytest` (or `uv run pytest`) | +| **Tear down test data** | `python teardown_test_data.py` | +| **Package script** | `run-evals` (installed via `[project.scripts]` in `pyproject.toml`) | + +### `--type` filter keywords (maps to pytest `-k` on case IDs) + +| `--type` | Matches IDs containing | +|----------|------------------------| +| `happy_path` | `happy` | +| `ambiguous_selection` | `ambiguous` | +| `missing_parameter` | `missing` | +| `multi_tool` | `multi_tool` | +| `multi_turn` | `clarify_then_call` | + +## CODING STANDARDS + +- **Language:** Python 3.11+ with full type hints (`typing`, `dataclasses`). +- **Style:** PEP 8; line length 120 (Ruff in `pyproject.toml`). +- **Docstrings:** Triple-quoted module- and function-level docstrings. +- **Error handling:** Exceptions caught and surfaced as readable messages (e.g. tool errors in `agent/client.py`). +- **Configuration:** `python-dotenv`; secrets in `.env` (never commit). Key vars: + - `MCP_SERVER_URL`, `EVALS_DATABASE`, `BEDROCK_MODEL_ID` + - `BEDROCK_JUDGE_MODEL_ID` (optional, defaults to agent model) + - `USE_DESCRIPTION_OVERRIDES` / `DESCRIPTION_OVERRIDES_FILE` (optional — default evals use live MCP descriptions) + - `AGENT_MAX_STEPS` (default 5, single-turn) + - `AGENT_MAX_STEPS_PER_TURN` (default 3, multi-turn) +- **Testing:** Pytest + DeepEval for integration evals; pure unit tests for `judge/checks.py` and schema validation without Bedrock/MCP. + +## WHERE TO LOOK + +| Task | Location | +|------|----------| +| Agent loop / MCP session | `agent/client.py` | +| Deterministic checks | `judge/checks.py` | +| LLM judge metrics | `judge/metrics.py` | +| Run + score any case | `tests/case_runner.py` → `assert_eval_case()` | +| Eval run summaries | `judge/report.py` → `results/runs//` + `results/latest.json` | +| List recent runs | `run_evals.py --list-runs` | +| Override suggestions | `suggest_overrides.py` → run's `suggested_overrides.json` | +| Apply override draft | `suggest_overrides.py --apply` → `description_overrides.json` | +| Apply overrides during evals | `description_overrides.json` + `--with-description-overrides` (`agent/client.py`) | +| Case JSON definitions | `cases/.json` | +| Ambiguous pair registry / live MCP diff | `backup/audit_cases.py` → `AMBIGUOUS_PAIRS`, `--live-mcp` | +| Happy-path generator | `backup/generate_cases.py` | +| Test data setup / preflight | `setup_test_data.py`, `preflight.py` | +| `{EVALS_DATABASE}` substitution | `tests/conftest.py` → `_substitute()` |s +| User-facing docs | `README.md`, `docs/` | + +## EVAL FLOW + +**Description quality loop** (see `docs/workflow.md`): + +1. Baseline: `run_evals.py` — agent uses live MCP descriptions. +2. Summary: `judge/report.py` writes `results/runs//` and updates `results/latest.json`. +3. Draft: `suggest_overrides.py` — LLM rewrites for failed `ambiguous_selection` cases. +4. Test: `suggest_overrides.py --apply`, then re-run with `--with-description-overrides`. +5. Ship: promote to MCP server, re-run baseline. + +**Per-case scoring:** + +1. Test loads case from `cases/.json` via `load_cases()`. +2. `{EVALS_DATABASE}` placeholder substituted at runtime. +3. **Single-turn:** `run_agent()` → deterministic checks → deepeval metrics. +4. **Multi-turn:** `run_agent_turns()` (one MCP session, max 7 turns) → per-turn checks: + - Clarification turns: no tools + Clarification GEval + - Tool turns: deterministic checks + ToolCorrectnessMetric +5. Structural failures in `judge/checks.py` raise `AssertionError` before the judge runs. +6. Outcomes recorded under `results/runs//` via `judge/report.py`. + +## CASE JSON CONVENTIONS + +**Single-turn:** top-level `input`, `type`, `expected_tools`. + +**Multi-turn:** top-level `turns` (2–7 entries), each with `input` and exactly one of: +- `"expect": "clarification"` +- `"expected_tools": [...]` + +Multi-turn case IDs should contain `clarify_then_call` for `--type multi_turn` filtering. + +Param names in `expected_tools` must match live MCP schemas (e.g. `sql` for `base_readQuery`, `user_name` for sec tools). Use `{EVALS_DATABASE}.evals_*` tables for deterministic grounding in base, qlty, plot, and dba cases where applicable. + +## MCP DESCRIPTION CHURN WORKFLOW + +When a tool description changes in the MCP server: + +1. `backup/generate_cases.py --module --overwrite` for happy paths (optional) +2. Manually review `ambiguous_selection` cases in that module +3. `backup/audit_cases.py --module --strict` (optional) +4. Run baseline evals; use `suggest_overrides.py` on failures; test with `--with-description-overrides` +5. Promote accepted wording to the MCP server repo; re-run baseline evals +6. Add new tool pairs to `AMBIGUOUS_PAIRS` in `backup/audit_cases.py` when overlap is discovered + +## NOTES + +- Requires running **Teradata MCP Server** at `MCP_SERVER_URL` and **ClearScape** database `EVALS_DATABASE` with demo tables from `setup_test_data.py`. +- AWS Bedrock credentials via standard boto3 chain. Default model: `anthropic.claude-3-5-sonnet-20241022-v2:0`. +- Eval results stored in `results/` (not committed). Each run gets `results/runs//` with summary, manifest, and optional suggestion draft. `results/latest.json` points at the newest run; `results/index.json` lists recent runs. +- Do not commit `.env` or credentials. +- Hand-author edge cases (`ambiguous_selection`, `missing_parameter`, `multi_tool`, `multi_turn`) after generating happy paths — generator cannot replace human judgement on description boundaries. +- Prompt vocabulary should differ from tool descriptions (same rule enforced in `backup/generate_cases.py` DRAFT_PROMPT). diff --git a/tests/eval_test/README.md b/tests/eval_test/README.md new file mode 100644 index 0000000..a624b7e --- /dev/null +++ b/tests/eval_test/README.md @@ -0,0 +1,39 @@ +# teradata-mcp-evals + +Eval suite for the [Teradata MCP Server](https://github.com/Teradata/teradata-mcp-server) community edition. + +Tests whether an LLM agent selects the right MCP tool and forms valid parameters from natural language. The primary goal is **MCP tool description quality** — routing failures on `ambiguous_selection` cases usually mean overlapping or unclear descriptions. Uses [deepeval](https://github.com/confident-ai/deepeval) and Claude on AWS Bedrock as agent and judge. + +## Quick start + +```bash +uv venv && uv sync +cp .env.example .env # set MCP_SERVER_URL, EVALS_DATABASE, Bedrock credentials + +python setup_test_data.py +python run_evals.py --module base +``` + +Open `results/latest_summary.md` for pass/fail details, or run `python run_evals.py --list-runs` to browse run directories. + +## Workflow + +Baseline evals use **live MCP descriptions**. To iterate on wording before changing the server: + +1. `run_evals.py` — baseline +2. `suggest_overrides.py` — LLM draft for failed cases +3. `suggest_overrides.py --apply` — merge reviewed draft into `description_overrides.json` +4. `run_evals.py --with-description-overrides` — test locally +5. Promote to MCP server → baseline again + +Full diagram, commands, and results files: **[docs/workflow.md](docs/workflow.md)** + +## Documentation + +| Doc | Contents | +|---|---| +| [docs/setup.md](docs/setup.md) | Install, `.env`, test data, unit tests | +| [docs/workflow.md](docs/workflow.md) | Running evals, overrides, results | +| [docs/cases.md](docs/cases.md) | Case types, JSON format, scoring, adding cases | +| [docs/structure.md](docs/structure.md) | Repository layout | +| [backup/README.md](backup/README.md) | Optional case generation and audit scripts | diff --git a/tests/eval_test/agent/__init__.py b/tests/eval_test/agent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/eval_test/agent/client.py b/tests/eval_test/agent/client.py new file mode 100644 index 0000000..9d93fed --- /dev/null +++ b/tests/eval_test/agent/client.py @@ -0,0 +1,372 @@ +"""Multi-turn MCP agent that drives tool selection via Bedrock Claude.""" + +from __future__ import annotations + +import asyncio +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import boto3 +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client + +MAX_TOOL_RESULT_CHARS = int(os.environ.get("MAX_TOOL_RESULT_CHARS", "8000")) + +# --------------------------------------------------------------------------- +# Description overrides (opt-in) +# --------------------------------------------------------------------------- +# By default evals use live MCP server tool descriptions (baseline). +# Set USE_DESCRIPTION_OVERRIDES=1 or pass --with-description-overrides to +# run_evals.py to patch descriptions from description_overrides.json before +# routing — useful for testing proposed wording before changing the MCP server. +# --------------------------------------------------------------------------- + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_OVERRIDES_FILE = REPO_ROOT / "description_overrides.json" + + +def description_overrides_enabled() -> bool: + """Return True when evals should patch tool descriptions before routing.""" + if os.environ.get("DESCRIPTION_OVERRIDES_FILE"): + return True + return os.environ.get("USE_DESCRIPTION_OVERRIDES", "").lower() in {"1", "true", "yes"} + + +def resolve_description_overrides_file() -> Path | None: + """Return the overrides file path when overrides are enabled.""" + if not description_overrides_enabled(): + return None + + env_path = os.environ.get("DESCRIPTION_OVERRIDES_FILE") + if env_path: + return Path(env_path) + + if DEFAULT_OVERRIDES_FILE.exists(): + return DEFAULT_OVERRIDES_FILE + + cwd_candidate = Path("description_overrides.json") + if cwd_candidate.exists(): + return cwd_candidate + + return DEFAULT_OVERRIDES_FILE + + +def get_description_override_status() -> dict[str, str | int | None]: + """Summarize which tool descriptions the agent sees during evals.""" + if not description_overrides_enabled(): + return {"mode": "mcp_server", "file": None, "tool_count": 0} + + overrides = _load_description_overrides() + overrides_file = resolve_description_overrides_file() + return { + "mode": "overrides", + "file": str(overrides_file) if overrides_file else None, + "tool_count": len(overrides), + } + + +def _load_description_overrides() -> dict[str, str]: + """Return {tool_name: description} from the overrides file, or {} if absent.""" + if not description_overrides_enabled(): + return {} + + overrides_file = resolve_description_overrides_file() + if overrides_file is None or not overrides_file.exists(): + return {} + + try: + data = json.loads(overrides_file.read_text()) + if isinstance(data, dict): + return {k: v for k, v in data.items() if isinstance(v, str)} + except Exception: + pass + return {} + + +def _apply_description_overrides(tools: list, overrides: dict[str, str]) -> list: + """Return a new list with tool descriptions replaced where an override exists.""" + if not overrides: + return tools + + patched = [] + for tool in tools: + name = getattr(tool, "name", None) + if name and name in overrides: + tool = tool.model_copy(update={"description": overrides[name]}) + patched.append(tool) + return patched + + +@dataclass +class ToolCallRecord: + name: str + input_parameters: dict[str, Any] + + +@dataclass +class TurnResult: + tool_calls: list[ToolCallRecord] + final_response: str + + +@dataclass +class AgentResult: + tool_calls: list[ToolCallRecord] + final_response: str + + +def _make_bedrock_client(region: str = "us-east-1"): + return boto3.client("bedrock-runtime", region_name=region) + + +def _mcp_tool_to_bedrock(tool) -> dict: + return { + "toolSpec": { + "name": tool.name, + "description": tool.description or "", + "inputSchema": {"json": tool.inputSchema}, + } + } + + +def _extract_text(content_blocks: list[dict]) -> str: + parts: list[str] = [] + for block in content_blocks: + if "text" in block: + parts.append(block["text"]) + elif block.get("type") == "text" and block.get("text"): + parts.append(block["text"]) + return "".join(parts) + + +def _iter_tool_uses(content_blocks: list[dict]): + for block in content_blocks: + if "toolUse" in block: + tool_use = block["toolUse"] + yield tool_use["name"], tool_use.get("input", {}), tool_use["toolUseId"] + elif block.get("type") == "toolUse": + yield block["name"], block.get("input", {}), block["toolUseId"] + + +async def _build_tool_results(session: ClientSession, content_blocks: list[dict]) -> tuple[list[dict], list[ToolCallRecord]]: + tool_results: list[dict] = [] + tool_calls: list[ToolCallRecord] = [] + + for tool_name, tool_input, tool_use_id in _iter_tool_uses(content_blocks): + tool_calls.append(ToolCallRecord(name=tool_name, input_parameters=tool_input)) + + try: + mcp_result = await session.call_tool(tool_name, tool_input) + result_text = json.dumps( + [c.model_dump() for c in mcp_result.content], + default=str, + ) + except Exception as exc: + result_text = f"Tool error: {exc}" + + if len(result_text) > MAX_TOOL_RESULT_CHARS: + result_text = result_text[:MAX_TOOL_RESULT_CHARS] + "\n... [truncated]" + + tool_results.append( + { + "toolResult": { + "toolUseId": tool_use_id, + "content": [{"text": result_text}], + } + } + ) + + return tool_results, tool_calls + + +async def _handle_tool_use( + session: ClientSession, + messages: list[dict], + output_message: dict, + *, + record_calls: list[ToolCallRecord], +) -> bool: + """Append tool results to messages. Returns False if tool_use had no usable blocks.""" + tool_results, tool_calls = await _build_tool_results(session, output_message.get("content", [])) + if not tool_results: + return False + + record_calls.extend(tool_calls) + messages.append({"role": "user", "content": tool_results}) + return True + +async def _run_agent_turns_async( + prompts: list[str], + model_id: str, + bedrock_client, + mcp_url: str, + max_steps_per_turn: int, +) -> list[TurnResult]: + """Run a scripted multi-turn conversation in one MCP session.""" + overrides = _load_description_overrides() + async with streamablehttp_client(mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + tools_response = await session.list_tools() + patched_tools = _apply_description_overrides(tools_response.tools, overrides) + bedrock_tools = [_mcp_tool_to_bedrock(t) for t in patched_tools] + + messages: list[dict] = [] + turn_results: list[TurnResult] = [] + + for prompt in prompts: + messages.append({"role": "user", "content": [{"text": prompt}]}) + turn_tool_calls: list[ToolCallRecord] = [] + final_response = "" + + for _ in range(max_steps_per_turn): + response = bedrock_client.converse( + modelId=model_id, + messages=messages, + toolConfig={ + "tools": bedrock_tools, + "toolChoice": {"auto": {}}, + }, + ) + + stop_reason = response["stopReason"] + output_message = response["output"]["message"] + messages.append(output_message) + + if stop_reason == "tool_use": + handled = await _handle_tool_use( + session, + messages, + output_message, + record_calls=turn_tool_calls, + ) + if not handled: + final_response = _extract_text(output_message.get("content", [])) + break + else: + final_response = _extract_text(output_message.get("content", [])) + break + + turn_results.append( + TurnResult(tool_calls=turn_tool_calls, final_response=final_response) + ) + + return turn_results + + +async def _run_agent_async( + prompt: str, + model_id: str, + bedrock_client, + mcp_url: str, + max_steps: int, +) -> AgentResult: + overrides = _load_description_overrides() + async with streamablehttp_client(mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + tools_response = await session.list_tools() + patched_tools = _apply_description_overrides(tools_response.tools, overrides) + bedrock_tools = [_mcp_tool_to_bedrock(t) for t in patched_tools] + + messages: list[dict] = [ + {"role": "user", "content": [{"text": prompt}]} + ] + tool_calls_made: list[ToolCallRecord] = [] + final_response = "" + + for _ in range(max_steps): + response = bedrock_client.converse( + modelId=model_id, + messages=messages, + toolConfig={ + "tools": bedrock_tools, + "toolChoice": {"auto": {}}, + }, + ) + + stop_reason = response["stopReason"] + output_message = response["output"]["message"] + messages.append(output_message) + + if stop_reason == "tool_use": + handled = await _handle_tool_use( + session, + messages, + output_message, + record_calls=tool_calls_made, + ) + if not handled: + final_response = _extract_text(output_message.get("content", [])) + break + else: + final_response = _extract_text(output_message.get("content", [])) + break + + return AgentResult(tool_calls=tool_calls_made, final_response=final_response) + + +def run_agent( + prompt: str, + model_id: str | None = None, + bedrock_client=None, + mcp_url: str | None = None, + max_steps: int | None = None, +) -> AgentResult: + """Synchronous entry point — runs the async agent loop via asyncio.run().""" + resolved_model = model_id or os.environ.get( + "BEDROCK_MODEL_ID", "anthropic.claude-3-5-sonnet-20241022-v2:0" + ) + resolved_url = mcp_url or os.environ.get("MCP_SERVER_URL", "http://127.0.0.1:8001/mcp") + resolved_steps = max_steps or int(os.environ.get("AGENT_MAX_STEPS", "5")) + + if bedrock_client is None: + bedrock_client = _make_bedrock_client( + region=os.environ.get("AWS_REGION", "us-east-1"), + ) + + return asyncio.run( + _run_agent_async( + prompt=prompt, + model_id=resolved_model, + bedrock_client=bedrock_client, + mcp_url=resolved_url, + max_steps=resolved_steps, + ) + ) + + +def run_agent_turns( + prompts: list[str], + model_id: str | None = None, + bedrock_client=None, + mcp_url: str | None = None, + max_steps_per_turn: int | None = None, +) -> list[TurnResult]: + """Run a shallow multi-turn conversation (separate tool-call budget per turn).""" + if not prompts: + raise ValueError("prompts must contain at least one user message") + + resolved_model = model_id or os.environ.get( + "BEDROCK_MODEL_ID", "anthropic.claude-3-5-sonnet-20241022-v2:0" + ) + resolved_url = mcp_url or os.environ.get("MCP_SERVER_URL", "http://127.0.0.1:8001/mcp") + resolved_steps = max_steps_per_turn or int(os.environ.get("AGENT_MAX_STEPS_PER_TURN", "3")) + + if bedrock_client is None: + bedrock_client = _make_bedrock_client( + region=os.environ.get("AWS_REGION", "us-east-1"), + ) + + return asyncio.run( + _run_agent_turns_async( + prompts=prompts, + model_id=resolved_model, + bedrock_client=bedrock_client, + mcp_url=resolved_url, + max_steps_per_turn=resolved_steps, + ) + ) diff --git a/tests/eval_test/backup/README.md b/tests/eval_test/backup/README.md new file mode 100644 index 0000000..0b6eebf --- /dev/null +++ b/tests/eval_test/backup/README.md @@ -0,0 +1,122 @@ +# Case generation, audit, and inspection utilities + +These scripts were used to bootstrap and maintain eval cases during initial development. They are **not part of the main eval workflow** — the case JSON under `cases/` is now the source of truth, and CI runs the core unit tests only. + +Use them when you need to draft new `happy_path` cases from live MCP tool descriptions, inspect live tool definitions, or check ambiguous-pair and live-tool coverage before an MCP server release. + +## Files + +| File | Purpose | +|---|---| +| `generate_cases.py` | Connects to the MCP server, reads tool descriptions, and asks Bedrock Claude to draft one `happy_path` case per tool into `cases/.json`. | +| `audit_cases.py` | Offline checks for registered ambiguous tool-pair coverage; optionally diffs case files against a live MCP server's tool list. | +| `inspect_tools.py` | Dumps live MCP tool names, descriptions, and input schemas to `results/live_descriptions_.json`. | +| `test_audit_cases.py` | Unit tests for the offline audit logic (no MCP or Bedrock required). | + +## Prerequisites + +Same as the main project: Python 3.11+, `.env` configured, and for most commands a running MCP server at `MCP_SERVER_URL`. `generate_cases.py` also requires AWS Bedrock access. + +## Generating happy path cases + +The `base`, `sec`, `chat`, and `plot` modules ship with hand-authored cases. For `dba`, `qlty`, and `tmpl`, you may want to generate happy paths from live tool descriptions first: + +```bash +# One module +uv run python backup/generate_cases.py --module dba + +# All in-scope modules +uv run python backup/generate_cases.py +``` + +Results are written to `cases/.json`. After generating, hand-author `ambiguous_selection`, `missing_parameter`, `multi_tool`, and (optionally) `multi_turn` cases — those require human judgement and cannot be generated cleanly from descriptions alone. + +### Generator flags + +```bash +# Preview without writing to disk +uv run python backup/generate_cases.py --dry-run + +# Overwrite existing happy_path cases (e.g. after a description change) +uv run python backup/generate_cases.py --overwrite +``` + +The generator enforces vocabulary **different from tool descriptions** in its draft prompt — the same rule used when hand-authoring edge cases. + +## Inspecting live tool descriptions + +Requires a running MCP server at `MCP_SERVER_URL`: + +```bash +# Default: base module only +uv run python backup/inspect_tools.py + +# One module +uv run python backup/inspect_tools.py --module dba + +# Every module on the server +uv run python backup/inspect_tools.py --all-modules +``` + +Output is written to `results/live_descriptions_.json` — useful when reviewing description wording before editing cases or MCP server tool definitions. + +## Auditing case coverage + +### Offline audit + +Checks registered ambiguous tool pairs in `cases/*.json` for base, dba, sec, and qlty: + +```bash +# Report gaps for priority modules +uv run python backup/audit_cases.py + +# Fail when pairs are missing coverage +uv run python backup/audit_cases.py --strict + +# One module only +uv run python backup/audit_cases.py --module base --strict +``` + +Registered pairs live in `audit_cases.py` (e.g. `base_readQuery` vs `base_tablePreview`). Extend that registry when you identify new description overlap between tools. + +### Live MCP diff (pre-release) + +Requires a running MCP server at `MCP_SERVER_URL`: + +```bash +# Diff live tool list vs cases — missing happy paths and stale tool names +uv run python backup/audit_cases.py --live-mcp + +# Fail on any gap (priority modules require happy_path for every live tool) +uv run python backup/audit_cases.py --live-mcp --strict + +# Live diff only, skip ambiguous pair checks +uv run python backup/audit_cases.py --live-mcp --skip-pairs --module base +``` + +| Check | Offline `--strict` | `--live-mcp --strict` | +|---|---|---| +| Ambiguous pair coverage | ✅ | ✅ (unless `--skip-pairs`) | +| Missing `happy_path` for live tools | — | ✅ (base, dba, sec, qlty) | +| Stale tool names in cases | — | ✅ | +| Non-priority modules (chat, plot) | — | stale names only; missing happy paths noted, not failed | + +## When MCP tool descriptions change + +1. **Regenerate happy paths** for the affected module: + ```bash + uv run python backup/generate_cases.py --module base --overwrite + ``` +2. **Review hand-authored cases** — especially `ambiguous_selection` prompts for that module. +3. **Run the audit** to confirm pair coverage and (optionally) sync with live tools: + ```bash + uv run python backup/audit_cases.py --module base --strict + uv run python backup/audit_cases.py --live-mcp --strict # requires running MCP server + ``` +4. **Run evals** and treat `ambiguous_selection` failures as description feedback. See [docs/workflow.md](../docs/workflow.md) for the baseline → suggest → override → promote loop. + +## Running unit tests + +```bash +uv run pytest backup/test_audit_cases.py -v +``` diff --git a/tests/eval_test/backup/__init__.py b/tests/eval_test/backup/__init__.py new file mode 100644 index 0000000..4d12f1c --- /dev/null +++ b/tests/eval_test/backup/__init__.py @@ -0,0 +1 @@ +"""Archived case generation and audit utilities (not part of the main eval workflow).""" diff --git a/tests/eval_test/backup/audit_cases.py b/tests/eval_test/backup/audit_cases.py new file mode 100644 index 0000000..78ab7e3 --- /dev/null +++ b/tests/eval_test/backup/audit_cases.py @@ -0,0 +1,291 @@ +""" +Audit eval case coverage: ambiguous tool pairs and (optionally) live MCP tool list. + +Usage: + uv run python backup/audit_cases.py # ambiguous pairs, all priority modules + uv run python backup/audit_cases.py --module base # one module + uv run python backup/audit_cases.py --strict # exit 1 on pair gaps (offline, CI-safe) + uv run python backup/audit_cases.py --strict --live-mcp # also diff against running MCP server +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + +CASES_DIR = Path(__file__).resolve().parent.parent / "cases" + +IN_SCOPE_MODULES = frozenset({"base", "dba", "sec", "qlty", "chat", "plot", "tmpl"}) +PRIORITY_MODULES = frozenset({"base", "dba", "sec", "qlty"}) +DEFAULT_AUDIT_MODULES = ("base", "dba", "sec", "qlty") + +# Tool pairs whose descriptions should be distinct; each needs ≥1 ambiguous_selection case. +AMBIGUOUS_PAIRS: dict[str, list[tuple[str, str]]] = { + "base": [ + ("base_readQuery", "base_tablePreview"), + ("base_tableDDL", "base_columnMetadata"), + ("base_tableList", "base_databaseList"), + ("base_saveDDL", "base_tableDDL"), + ("base_tableAffinity", "base_tableUsage"), + ("base_columnDescription", "base_columnMetadata"), + ], + "dba": [ + ("dba_tableSpace", "dba_databaseSpace"), + ("dba_databaseSpace", "dba_systemSpace"), + ("dba_tableSqlList", "dba_userSqlList"), + ("dba_tableUsageImpact", "dba_resusageSummary"), + ("dba_tableUsageImpact", "dba_sessionInfo"), + ("dba_userDelay", "dba_flowControl"), + ], + "sec": [ + ("sec_userDbPermissions", "sec_userRoles"), + ("sec_userDbPermissions", "sec_rolePermissions"), + ("sec_userRoles", "sec_rolePermissions"), + ], + "qlty": [ + ("qlty_missingValues", "qlty_rowsWithMissingValues"), + ("qlty_standardDeviation", "qlty_univariateStatistics"), + ("qlty_columnSummary", "qlty_univariateStatistics"), + ], +} + + +def get_module(tool_name: str) -> str | None: + prefix = tool_name.split("_")[0] + return prefix if prefix in IN_SCOPE_MODULES else None + + +def load_cases(module: str) -> list[dict]: + path = CASES_DIR / f"{module}.json" + if not path.exists(): + return [] + data = json.loads(path.read_text()) + return [c for c in data.get("cases", []) if "id" in c] + + +def tools_referenced_in_cases(cases: list[dict]) -> set[str]: + names: set[str] = set() + for case in cases: + for block in case.get("expected_tools", []): + names.add(block["name"]) + for turn in case.get("turns", []): + for block in turn.get("expected_tools", []): + names.add(block["name"]) + return names + + +def happy_path_tools(cases: list[dict]) -> set[str]: + names: set[str] = set() + for case in cases: + if case.get("type") != "happy_path": + continue + for block in case.get("expected_tools", []): + names.add(block["name"]) + return names + + +def covered_ambiguous_pairs(module: str, cases: list[dict]) -> set[frozenset[str]]: + pairs = AMBIGUOUS_PAIRS.get(module, []) + if not pairs: + return set() + + covered: set[frozenset[str]] = set() + for case in cases: + if case.get("type") != "ambiguous_selection": + continue + expected_tools = case.get("expected_tools", []) + if not expected_tools: + continue + winner = expected_tools[0]["name"] + case_id = case.get("id", "") + description = case.get("description", "") + + for tool_a, tool_b in pairs: + pair = frozenset({tool_a, tool_b}) + if winner not in pair: + continue + loser = tool_b if winner == tool_a else tool_a + loser_token = loser.split("_", 1)[-1] + if loser_token in case_id or loser in description: + covered.add(pair) + + return covered + + +def audit_ambiguous_pair_gaps(module: str, cases: list[dict]) -> list[str]: + pairs = AMBIGUOUS_PAIRS.get(module, []) + if not pairs: + return [] + + covered = covered_ambiguous_pairs(module, cases) + gaps: list[str] = [] + happy_tools = happy_path_tools(cases) + + for tool_a, tool_b in pairs: + pair = frozenset({tool_a, tool_b}) + if pair not in covered: + gaps.append(f" missing ambiguous case: {tool_a} vs {tool_b}") + + for tool_a, tool_b in pairs: + for tool in (tool_a, tool_b): + if tool not in happy_tools: + gaps.append(f" missing happy_path for tool involved in pair: {tool}") + + return gaps + + +def audit_live_tool_gaps( + module: str, + live_tools: set[str], + cases: list[dict], + *, + require_happy_path: bool, +) -> list[str]: + gaps: list[str] = [] + happy_tools = happy_path_tools(cases) + referenced = tools_referenced_in_cases(cases) + + if require_happy_path: + for tool in sorted(live_tools): + if tool not in happy_tools: + gaps.append(f" missing happy_path for live tool: {tool}") + + for tool in sorted(referenced): + if tool not in live_tools: + gaps.append(f" stale tool name in cases (not on MCP server): {tool}") + + extra_live = live_tools - referenced + if extra_live and not require_happy_path: + for tool in sorted(extra_live): + if tool not in happy_tools: + gaps.append(f" note: live tool has no cases yet: {tool}") + + return gaps + + +async def fetch_tools_by_module(mcp_url: str) -> dict[str, set[str]]: + from mcp import ClientSession + from mcp.client.streamable_http import streamablehttp_client + + async with streamablehttp_client(mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.list_tools() + + by_module: dict[str, set[str]] = {module: set() for module in IN_SCOPE_MODULES} + for tool in result.tools: + module = get_module(tool.name) + if module: + by_module[module].add(tool.name) + return by_module + + +def audit_module( + module: str, + *, + live_tools: set[str] | None = None, + check_pairs: bool = True, + check_live: bool = False, +) -> list[str]: + cases = load_cases(module) + gaps: list[str] = [] + + if check_pairs: + gaps.extend(audit_ambiguous_pair_gaps(module, cases)) + + if check_live and live_tools is not None: + require_happy = module in PRIORITY_MODULES + gaps.extend( + audit_live_tool_gaps( + module, + live_tools, + cases, + require_happy_path=require_happy, + ) + ) + + return gaps + + +def resolve_modules(module: str | None, live_mcp: bool) -> list[str]: + if module: + return [module] + if live_mcp: + return sorted(IN_SCOPE_MODULES) + return list(DEFAULT_AUDIT_MODULES) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Audit eval case coverage") + parser.add_argument("--module", help="Audit one module only") + parser.add_argument( + "--live-mcp", + action="store_true", + help="Diff cases against tools returned by the running MCP server", + ) + parser.add_argument( + "--skip-pairs", + action="store_true", + help="Skip ambiguous pair checks (live MCP diff only)", + ) + parser.add_argument("--strict", action="store_true", help="Exit 1 if any gaps found") + args = parser.parse_args() + + modules = resolve_modules(args.module, args.live_mcp) + live_by_module: dict[str, set[str]] | None = None + + if args.live_mcp: + mcp_url = os.environ.get("MCP_SERVER_URL", "http://127.0.0.1:8001/mcp") + print(f"Connecting to MCP server at {mcp_url} ...") + try: + live_by_module = asyncio.run(fetch_tools_by_module(mcp_url)) + total = sum(len(tools) for tools in live_by_module.values()) + print(f"Found {total} in-scope tools across {len(IN_SCOPE_MODULES)} modules\n") + except Exception as exc: + print(f"ERROR: could not connect to MCP server: {exc}", file=sys.stderr) + sys.exit(1) + + any_gaps = False + check_pairs = not args.skip_pairs + + for module in modules: + live_tools = live_by_module.get(module, set()) if live_by_module else None + gaps = audit_module( + module, + live_tools=live_tools, + check_pairs=check_pairs and module in AMBIGUOUS_PAIRS, + check_live=args.live_mcp, + ) + + print(f"[{module}]") + if not gaps: + if args.live_mcp and live_tools is not None: + print(f" OK — {len(live_tools)} live tool(s), cases in sync") + elif check_pairs and module in AMBIGUOUS_PAIRS: + print(" OK — all registered ambiguous pairs covered") + else: + print(" OK") + else: + any_gaps = True + print(" GAPS:") + for gap in gaps: + print(gap) + print() + + if any_gaps and args.strict: + sys.exit(1) + + if not any_gaps: + print("Audit passed.") + + +if __name__ == "__main__": + main() diff --git a/tests/eval_test/backup/generate_cases.py b/tests/eval_test/backup/generate_cases.py new file mode 100644 index 0000000..97719dc --- /dev/null +++ b/tests/eval_test/backup/generate_cases.py @@ -0,0 +1,175 @@ +""" +Bootstrap generator: connects to the MCP server, reads tool descriptions, +and asks Bedrock Claude to draft one happy_path test case per tool. + +Usage: + uv run python backup/generate_cases.py # all in-scope modules + uv run python backup/generate_cases.py --module base # one module only + uv run python backup/generate_cases.py --dry-run # print without writing +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from pathlib import Path + +import boto3 +from dotenv import load_dotenv +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client + +load_dotenv() + +CASES_DIR = Path(__file__).resolve().parent.parent / "cases" +IN_SCOPE_MODULES = {"base", "dba", "sec", "qlty", "chat", "plot", "tmpl"} + +DRAFT_PROMPT = """\ +You are building an eval suite for an MCP tool server connected to a Teradata database. + +Below is the definition of one MCP tool: + +Name: {name} +Description: {description} +Input schema: {schema} + +Write ONE natural language prompt that a business user might type to invoke this tool. +Rules: +- Do NOT copy phrases directly from the tool description — use different vocabulary. +- Use concrete but generic values (e.g. a real-sounding database name like "Sales" or "HR"). +- The prompt should be unambiguous: it should clearly call for THIS tool, not a different one. +- Keep it to one or two sentences. + +Also provide the expected parameter values the tool should be called with. +Use realistic placeholder values where the actual value depends on environment. + +Respond with valid JSON only — no markdown, no explanation: +{{ + "input": "", + "expected_params": {{: , ...}} +}} +""" + + +def get_module(tool_name: str) -> str | None: + prefix = tool_name.split("_")[0] + return prefix if prefix in IN_SCOPE_MODULES else None + + +def make_bedrock_client(): + region = os.environ.get("AWS_REGION", "us-east-1") + return boto3.client("bedrock-runtime", region_name=region) + + +def draft_case(tool, bedrock_client, model_id: str) -> dict | None: + prompt = DRAFT_PROMPT.format( + name=tool.name, + description=tool.description or "(no description)", + schema=json.dumps(tool.inputSchema, indent=2), + ) + try: + response = bedrock_client.converse( + modelId=model_id, + messages=[{"role": "user", "content": [{"text": prompt}]}], + ) + text = response["output"]["message"]["content"][0]["text"].strip() + # Extract JSON even if there's surrounding whitespace or markdown fences + start = text.index("{") + end = text.rindex("}") + 1 + parsed = json.loads(text[start:end]) + return { + "id": f"{tool.name}_happy", + "type": "happy_path", + "description": f"Auto-generated happy path for {tool.name}", + "input": parsed["input"], + "expected_tools": [ + {"name": tool.name, "params": parsed.get("expected_params", {})} + ], + } + except Exception as exc: + print(f" WARNING: could not generate case for {tool.name}: {exc}", file=sys.stderr) + return None + + +async def collect_tools(mcp_url: str) -> list: + async with streamablehttp_client(mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.list_tools() + return result.tools + + +def load_existing(module: str) -> list[dict]: + path = CASES_DIR / f"{module}.json" + if not path.exists(): + return [] + data = json.loads(path.read_text()) + return [c for c in data.get("cases", []) if "id" in c] + + +def write_cases(module: str, cases: list[dict]) -> None: + path = CASES_DIR / f"{module}.json" + path.write_text(json.dumps({"module": module, "cases": cases}, indent=2) + "\n") + print(f" Wrote {len(cases)} cases → {path}") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate happy_path eval cases from live MCP tool descriptions") + parser.add_argument("--module", help="Only generate for this module (default: all in-scope)") + parser.add_argument("--dry-run", action="store_true", help="Print generated cases without writing to disk") + parser.add_argument("--overwrite", action="store_true", help="Replace existing happy_path cases (default: skip if already present)") + args = parser.parse_args() + + mcp_url = os.environ.get("MCP_SERVER_URL", "http://127.0.0.1:8001/mcp") + model_id = os.environ.get("BEDROCK_MODEL_ID", "anthropic.claude-3-5-sonnet-20241022-v2:0") + target_modules = {args.module} if args.module else IN_SCOPE_MODULES + + print(f"Connecting to MCP server at {mcp_url} ...") + tools = asyncio.run(collect_tools(mcp_url)) + print(f"Found {len(tools)} tools total") + + bedrock = make_bedrock_client() + + # Group tools by module + by_module: dict[str, list] = {m: [] for m in target_modules} + for tool in tools: + mod = get_module(tool.name) + if mod in target_modules: + by_module[mod].append(tool) + + for module, module_tools in by_module.items(): + if not module_tools: + print(f"\n[{module}] no tools found — skipping") + continue + + print(f"\n[{module}] {len(module_tools)} tools") + existing = load_existing(module) + existing_ids = {c["id"] for c in existing} + new_cases = list(existing) + + for tool in module_tools: + case_id = f"{tool.name}_happy" + if case_id in existing_ids and not args.overwrite: + print(f" Skipping {tool.name} (already has happy_path case)") + continue + print(f" Drafting case for {tool.name} ...") + case = draft_case(tool, bedrock, model_id) + if case is None: + continue + # Replace existing happy_path if overwrite, else append + new_cases = [c for c in new_cases if c["id"] != case_id] + new_cases.append(case) + + if args.dry_run: + print(json.dumps({"module": module, "cases": new_cases}, indent=2)) + else: + write_cases(module, new_cases) + + print("\nDone. Review generated cases, then hand-author ambiguous_selection, missing_parameter, and multi_tool cases.") + + +if __name__ == "__main__": + main() diff --git a/tests/eval_test/backup/inspect_tools.py b/tests/eval_test/backup/inspect_tools.py new file mode 100644 index 0000000..10ebb39 --- /dev/null +++ b/tests/eval_test/backup/inspect_tools.py @@ -0,0 +1,92 @@ +""" +Dump live MCP tool descriptions to results/live_descriptions_.json. + +Usage: + uv run python backup/inspect_tools.py # all base tools + uv run python backup/inspect_tools.py --module base # explicit module filter + uv run python backup/inspect_tools.py --all-modules # every module on the server + +Output: results/live_descriptions_.json (one file per module) +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import os +from pathlib import Path + +from dotenv import load_dotenv +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client + +load_dotenv() + +RESULTS_DIR = Path(__file__).resolve().parent.parent / "results" +DEFAULT_MODULES = {"base"} + + +async def fetch_tools(mcp_url: str) -> list[dict]: + async with streamablehttp_client(mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.list_tools() + return [ + { + "name": t.name, + "description": t.description or "", + "inputSchema": t.inputSchema, + } + for t in result.tools + ] + + +def group_by_module(tools: list[dict]) -> dict[str, list[dict]]: + grouped: dict[str, list[dict]] = {} + for tool in tools: + module = tool["name"].split("_")[0] + grouped.setdefault(module, []).append(tool) + return grouped + + +def main() -> None: + parser = argparse.ArgumentParser(description="Dump live MCP tool descriptions.") + parser.add_argument("--module", default=None, help="Filter to a single module prefix (e.g. base)") + parser.add_argument("--all-modules", action="store_true", help="Include all modules (default: base only)") + args = parser.parse_args() + + mcp_url = os.environ.get("MCP_SERVER_URL", "http://127.0.0.1:8001/mcp") + print(f"Fetching tools from {mcp_url} ...") + + tools = asyncio.run(fetch_tools(mcp_url)) + grouped = group_by_module(tools) + + if args.module: + target_modules = {args.module} + elif args.all_modules: + target_modules = set(grouped.keys()) + else: + target_modules = DEFAULT_MODULES + + RESULTS_DIR.mkdir(exist_ok=True) + + for module, module_tools in sorted(grouped.items()): + if module not in target_modules: + continue + + out_path = RESULTS_DIR / f"live_descriptions_{module}.json" + payload = { + "module": module, + "mcp_url": mcp_url, + "tool_count": len(module_tools), + "tools": module_tools, + } + out_path.write_text(json.dumps(payload, indent=2)) + print(f" [{module}] {len(module_tools)} tools → {out_path}") + + print("Done.") + + +if __name__ == "__main__": + main() diff --git a/tests/eval_test/cases/base.json b/tests/eval_test/cases/base.json new file mode 100644 index 0000000..e72afb6 --- /dev/null +++ b/tests/eval_test/cases/base.json @@ -0,0 +1,460 @@ +{ + "module": "base", + "cases": [ + { + "id": "base_readQuery_happy", + "type": "happy_path", + "description": "Execute a SQL query against the evals employees table", + "input": "Run this query: SELECT name, department, salary FROM {EVALS_DATABASE}.evals_employees WHERE department = 'Sales'", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT name, department, salary FROM {EVALS_DATABASE}.evals_employees WHERE department = 'Sales'" + } + } + ] + }, + { + "id": "base_readQuery_ambiguous_vs_tablePreview", + "type": "ambiguous_selection", + "description": "Prompt with a WHERE clause should use base_readQuery, not base_tablePreview", + "input": "Get me all orders from {EVALS_DATABASE}.evals_orders where the amount is greater than 500", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT * FROM {EVALS_DATABASE}.evals_orders WHERE amount > 500" + } + } + ] + }, + { + "id": "base_readQuery_missing_sql", + "type": "missing_parameter", + "description": "Agent must ask for clarification when no SQL or table is given", + "input": "Query the database for me", + "expected_tools": [] + }, + { + "id": "base_tableList_happy", + "type": "happy_path", + "description": "List tables in the evals database", + "input": "What tables are in the {EVALS_DATABASE} database?", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + }, + { + "id": "base_tableList_missing_database", + "type": "missing_parameter", + "description": "User asks for tables without specifying a database", + "input": "Show me what tables are available", + "expected_tools": [] + }, + { + "id": "base_tableDDL_happy", + "type": "happy_path", + "description": "Retrieve DDL for the evals employees table", + "input": "Show me the CREATE statement for evals_employees in the {EVALS_DATABASE} database", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_employees" + } + } + ] + }, + { + "id": "base_tableDDL_ambiguous_vs_columnMetadata", + "type": "ambiguous_selection", + "description": "Asking for full structure including constraints should prefer base_tableDDL", + "input": "Describe the full structure of {EVALS_DATABASE}.evals_employees including any constraints and indexes", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_employees" + } + } + ] + }, + { + "id": "base_columnMetadata_happy", + "type": "happy_path", + "description": "Get precise Teradata type codes, character sets, and precision for columns in evals_orders", + "input": "Give me the exact Teradata type codes, character set, and decimal precision for every column in {EVALS_DATABASE}.evals_orders", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "{EVALS_DATABASE}", + "object_name": "evals_orders" + } + } + ] + }, + { + "id": "base_columnMetadata_ambiguous_vs_tableDDL", + "type": "ambiguous_selection", + "description": "Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL", + "input": "I need the UNICODE vs LATIN character set and nullability for every column in {EVALS_DATABASE}.evals_employees", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "{EVALS_DATABASE}", + "object_name": "evals_employees" + } + } + ] + }, + { + "id": "base_tablePreview_happy", + "type": "happy_path", + "description": "Preview a sample of rows from the evals orders table", + "input": "Give me a quick preview of what's in {EVALS_DATABASE}.evals_orders", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_orders" + } + } + ] + }, + { + "id": "base_tablePreview_ambiguous_vs_readQuery", + "type": "ambiguous_selection", + "description": "A simple 'show me a few rows' without a WHERE clause should use base_tablePreview", + "input": "Show me a few rows from {EVALS_DATABASE}.evals_orders", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_orders" + } + } + ] + }, + { + "id": "base_databaseList_happy", + "type": "happy_path", + "description": "List all accessible databases", + "input": "What databases do I have access to?", + "expected_tools": [ + { + "name": "base_databaseList", + "params": {} + } + ] + }, + { + "id": "base_tableList_ambiguous_vs_databaseList", + "type": "ambiguous_selection", + "description": "Listing tables inside one database should prefer base_tableList, not base_databaseList", + "input": "What tables exist inside the {EVALS_DATABASE} database?", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + }, + { + "id": "base_databaseList_ambiguous_vs_tableList", + "type": "ambiguous_selection", + "description": "Listing accessible databases should prefer base_databaseList, not base_tableList", + "input": "Which databases am I allowed to connect to on this system?", + "expected_tools": [ + { + "name": "base_databaseList", + "params": {} + } + ] + }, + { + "id": "base_saveDDL_ambiguous_vs_tableDDL", + "type": "ambiguous_selection", + "description": "Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL", + "input": "Export the DDL for {EVALS_DATABASE}.evals_orders to a file on disk", + "expected_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_orders" + } + } + ] + }, + { + "id": "base_tableAffinity_ambiguous_vs_tableUsage", + "type": "ambiguous_selection", + "description": "Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage", + "input": "Which tables in {EVALS_DATABASE} tend to appear in the same queries as evals_employees?", + "expected_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_employees" + } + } + ] + }, + { + "id": "base_tableUsage_ambiguous_vs_tableAffinity", + "type": "ambiguous_selection", + "description": "Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity", + "input": "Who has been reading tables in {EVALS_DATABASE} lately and how often?", + "expected_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + }, + { + "id": "base_saveDDL_happy", + "type": "happy_path", + "description": "Save DDL for the evals employees table to a file", + "input": "Save the DDL for evals_employees in {EVALS_DATABASE} to a file", + "expected_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_employees" + } + } + ] + }, + { + "id": "base_tableAffinity_happy", + "type": "happy_path", + "description": "Find tables commonly queried together with evals_orders", + "input": "Which tables in {EVALS_DATABASE} are most often queried together with evals_orders?", + "expected_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_orders" + } + } + ] + }, + { + "id": "base_tableUsage_happy", + "type": "happy_path", + "description": "Check usage metrics for the evals database", + "input": "How frequently are the tables in {EVALS_DATABASE} being accessed and by whom?", + "expected_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + }, + { + "id": "base_multi_tool_list_then_preview", + "type": "multi_tool", + "description": "List tables in evals database then preview one", + "input": "List all tables in {EVALS_DATABASE}, then show me a preview of evals_employees", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "{EVALS_DATABASE}" + } + }, + { + "name": "base_tablePreview", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_employees" + } + } + ] + }, + { + "id": "base_multi_tool_ddl_then_columns", + "type": "multi_tool", + "description": "Get DDL then drill into column detail for the same table", + "input": "First show me the DDL for {EVALS_DATABASE}.evals_orders, then give me the detailed column metadata", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_orders" + } + }, + { + "name": "base_columnMetadata", + "params": { + "database_name": "{EVALS_DATABASE}", + "object_name": "evals_orders" + } + } + ] + }, + { + "id": "base_tableDDL_missing_table", + "type": "missing_parameter", + "description": "Agent must ask for database name when only table is provided", + "input": "Show me the CREATE statement for evals_employees", + "expected_tools": [] + }, + { + "id": "base_tablePreview_missing_table", + "type": "missing_parameter", + "description": "Agent must ask which table to preview", + "input": "Preview some rows", + "expected_tools": [] + }, + { + "id": "base_saveDDL_missing_table", + "type": "missing_parameter", + "description": "Agent must ask which table to save DDL for", + "input": "Save the DDL to a file", + "expected_tools": [] + }, + { + "id": "base_tablePreview_clarify_then_call", + "type": "missing_parameter", + "description": "Agent asks which table to preview, then calls base_tablePreview after user clarifies", + "turns": [ + { + "input": "Preview some rows for me", + "expect": "clarification" + }, + { + "input": "Preview rows from {EVALS_DATABASE}.evals_employees", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_employees" + } + } + ] + } + ] + }, + { + "id": "base_tableList_clarify_then_call", + "type": "missing_parameter", + "description": "Agent asks which database, then calls base_tableList after user clarifies", + "turns": [ + { + "input": "Show me what tables are available", + "expect": "clarification" + }, + { + "input": "List tables in the {EVALS_DATABASE} database", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + } + ] + }, + { + "id": "base_readQuery_clarify_then_call", + "type": "missing_parameter", + "description": "Agent asks for SQL, then calls base_readQuery after user provides the query", + "turns": [ + { + "input": "Run a query for me", + "expect": "clarification" + }, + { + "input": "Run: SELECT name, department FROM {EVALS_DATABASE}.evals_employees WHERE department = 'Sales'", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT name, department FROM {EVALS_DATABASE}.evals_employees WHERE department = 'Sales'" + } + } + ] + } + ] + }, + { + "id": "base_columnDescription_happy", + "type": "happy_path", + "description": "Show basic column info for the evals employees table", + "input": "Show me the columns in {EVALS_DATABASE}.evals_employees", + "expected_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_employees" + } + } + ] + }, + { + "id": "base_columnDescription_ambiguous_vs_columnMetadata", + "type": "ambiguous_selection", + "description": "A simple 'what columns does this table have' question should prefer base_columnDescription", + "input": "What are the column names and types in {EVALS_DATABASE}.evals_orders?", + "expected_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_orders" + } + } + ] + }, + { + "id": "base_columnMetadata_ambiguous_vs_columnDescription", + "type": "ambiguous_selection", + "description": "Asking for precise Teradata type codes and character sets should prefer base_columnMetadata", + "input": "Give me the exact Teradata type codes, character sets, and precision details for every column in {EVALS_DATABASE}.evals_employees", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "{EVALS_DATABASE}", + "object_name": "evals_employees" + } + } + ] + }, + { + "id": "base_unknown_tool_missing", + "type": "missing_parameter", + "description": "User asks to do something no tool supports — agent should explain and ask for clarification", + "input": "Automatically tune my Teradata database indexes and rebalance the AMPs", + "expected_tools": [] + } + ] +} diff --git a/tests/eval_test/cases/chat.json b/tests/eval_test/cases/chat.json new file mode 100644 index 0000000..005c8a2 --- /dev/null +++ b/tests/eval_test/cases/chat.json @@ -0,0 +1,19 @@ +{ + "module": "chat", + "cases": [ + { + "id": "chat_completeChat_missing_message", + "type": "missing_parameter", + "description": "User invokes chat without providing a message", + "input": "Can you use the chat completion tool for me?", + "expected_tools": [] + }, + { + "id": "chat_unknown_tool_missing", + "type": "missing_parameter", + "description": "User asks for a non‑existent chat tool", + "input": "Talk to the assistant about budgeting", + "expected_tools": [] + } + ] +} diff --git a/tests/eval_test/cases/dba.json b/tests/eval_test/cases/dba.json new file mode 100644 index 0000000..b6ba1ce --- /dev/null +++ b/tests/eval_test/cases/dba.json @@ -0,0 +1,441 @@ +{ + "module": "dba", + "cases": [ + { + "id": "dba_tableSpace_happy", + "type": "happy_path", + "description": "List largest tables by storage in the evals database", + "input": "Show me the 10 largest tables in {EVALS_DATABASE} ranked by storage consumption.", + "expected_tools": [ + { + "name": "dba_tableSpace", + "params": { + "database_name": "{EVALS_DATABASE}", + "top_n": 10 + } + } + ] + }, + { + "id": "dba_tableSqlList_happy", + "type": "happy_path", + "description": "List SQL executed against evals_orders over the past 14 days", + "input": "Show me all queries run against evals_orders over the last 14 days.", + "expected_tools": [ + { + "name": "dba_tableSqlList", + "params": { + "table_name": "evals_orders", + "no_days": 14 + } + } + ] + }, + { + "id": "dba_userSqlList_happy", + "type": "happy_path", + "description": "Show queries executed by a specific user over the past 14 days", + "input": "Show me what queries John_Smith has been executing over the past 14 days.", + "expected_tools": [ + { + "name": "dba_userSqlList", + "params": { + "user_name": "John_Smith", + "no_days": 14 + } + } + ] + }, + { + "id": "dba_databaseSpace_happy", + "type": "happy_path", + "description": "Storage allocated to the evals database", + "input": "How much disk space is allocated to the {EVALS_DATABASE} database?", + "expected_tools": [ + { + "name": "dba_databaseSpace", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + }, + { + "id": "dba_tableUsageImpact_happy", + "type": "happy_path", + "description": "Which users are consuming the most resources in the evals database", + "input": "Which users are consuming the most resources in {EVALS_DATABASE}?", + "expected_tools": [ + { + "name": "dba_tableUsageImpact", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + }, + { + "id": "dba_resusageSummary_happy", + "type": "happy_path", + "description": "System-wide CPU/memory/IO by application over the past month", + "input": "Show me how much CPU, memory, and I/O was consumed over the past month, grouped by application.", + "expected_tools": [ + { + "name": "dba_resusageSummary", + "params": { + "no_days": 30 + } + } + ] + }, + { + "id": "dba_databaseVersion_happy", + "type": "happy_path", + "description": "Get current Teradata software version", + "input": "What release of Teradata software is currently running on our system?", + "expected_tools": [ + { + "name": "dba_databaseVersion", + "params": {} + } + ] + }, + { + "id": "dba_flowControl_happy", + "type": "happy_path", + "description": "System throttling and delay statistics for a date range", + "input": "Show me the system throttling and flow control delay statistics from January 15th 2024 through February 28th 2024.", + "expected_tools": [ + { + "name": "dba_flowControl", + "params": { + "start_date": "2024-01-15", + "end_date": "2024-02-28" + } + } + ] + }, + { + "id": "dba_featureUsage_happy", + "type": "happy_path", + "description": "Feature utilization metrics for a date range", + "input": "Show me how different Teradata features were being utilized by our users between January 15th, 2024 and March 31st, 2024.", + "expected_tools": [ + { + "name": "dba_featureUsage", + "params": { + "start_date": "2024-01-15", + "end_date": "2024-03-31" + } + } + ] + }, + { + "id": "dba_userDelay_happy", + "type": "happy_path", + "description": "How long users waited in queues for a date range", + "input": "Show me how long users were waiting in the query queue between March 1st 2024 and March 31st 2024.", + "expected_tools": [ + { + "name": "dba_userDelay", + "params": { + "start_date": "2024-03-01", + "end_date": "2024-03-31" + } + } + ] + }, + { + "id": "dba_sessionInfo_happy", + "type": "happy_path", + "description": "Active sessions for a specific user", + "input": "Show me what active connections the database user SALES_ADMIN currently has open in Teradata.", + "expected_tools": [ + { + "name": "dba_sessionInfo", + "params": { + "user_name": "SALES_ADMIN" + } + } + ] + }, + { + "id": "dba_systemSpace_happy", + "type": "happy_path", + "description": "Total disk consumption across all databases", + "input": "How much disk capacity is currently being consumed across all databases in our data warehouse?", + "expected_tools": [ + { + "name": "dba_systemSpace", + "params": {} + } + ] + }, + { + "id": "dba_tableSpace_ambiguous_vs_databaseSpace", + "type": "ambiguous_selection", + "description": "Ranking tables by size inside one database should prefer dba_tableSpace, not dba_databaseSpace", + "input": "Which tables in {EVALS_DATABASE} are using the most disk space?", + "expected_tools": [ + { + "name": "dba_tableSpace", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + }, + { + "id": "dba_databaseSpace_ambiguous_vs_systemSpace", + "type": "ambiguous_selection", + "description": "Storage for one named database should prefer dba_databaseSpace, not dba_systemSpace", + "input": "How much total space is the {EVALS_DATABASE} database using?", + "expected_tools": [ + { + "name": "dba_databaseSpace", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + }, + { + "id": "dba_systemSpace_ambiguous_vs_databaseSpace", + "type": "ambiguous_selection", + "description": "Warehouse-wide storage across all databases should prefer dba_systemSpace, not dba_databaseSpace", + "input": "What is the total disk usage across the entire Teradata system?", + "expected_tools": [ + { + "name": "dba_systemSpace", + "params": {} + } + ] + }, + { + "id": "dba_tableSqlList_ambiguous_vs_userSqlList", + "type": "ambiguous_selection", + "description": "Queries run against a table should prefer dba_tableSqlList, not dba_userSqlList", + "input": "What SQL has been executed against evals_orders in the past week?", + "expected_tools": [ + { + "name": "dba_tableSqlList", + "params": { + "table_name": "evals_orders", + "no_days": 7 + } + } + ] + }, + { + "id": "dba_userSqlList_ambiguous_vs_tableSqlList", + "type": "ambiguous_selection", + "description": "Queries run by a specific user should prefer dba_userSqlList, not dba_tableSqlList", + "input": "List all SQL statements that user ANALYTICS_USER has run over the past 7 days.", + "expected_tools": [ + { + "name": "dba_userSqlList", + "params": { + "user_name": "ANALYTICS_USER", + "no_days": 7 + } + } + ] + }, + { + "id": "dba_tableUsageImpact_ambiguous_vs_resusageSummary", + "type": "ambiguous_selection", + "description": "Heavy users of a specific database should prefer dba_tableUsageImpact, not dba_resusageSummary", + "input": "Who is generating the heaviest workload against {EVALS_DATABASE}?", + "expected_tools": [ + { + "name": "dba_tableUsageImpact", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + }, + { + "id": "dba_resusageSummary_ambiguous_vs_tableUsageImpact", + "type": "ambiguous_selection", + "description": "System-wide CPU/memory/I/O by application should prefer dba_resusageSummary, not dba_tableUsageImpact", + "input": "Break down CPU, memory, and I/O consumption by application over the last 30 days.", + "expected_tools": [ + { + "name": "dba_resusageSummary", + "params": { + "no_days": 30 + } + } + ] + }, + { + "id": "dba_tableUsageImpact_ambiguous_vs_sessionInfo", + "type": "ambiguous_selection", + "description": "Historical resource consumption by users in a database should prefer dba_tableUsageImpact, not dba_sessionInfo", + "input": "Which users have been consuming the most CPU and query resources in {EVALS_DATABASE} over time?", + "expected_tools": [ + { + "name": "dba_tableUsageImpact", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + }, + { + "id": "dba_sessionInfo_ambiguous_vs_tableUsageImpact", + "type": "ambiguous_selection", + "description": "Open sessions for a specific user should prefer dba_sessionInfo, not dba_tableUsageImpact", + "input": "What active sessions does user SALES_ADMIN currently have open?", + "expected_tools": [ + { + "name": "dba_sessionInfo", + "params": { + "user_name": "SALES_ADMIN" + } + } + ] + }, + { + "id": "dba_userDelay_ambiguous_vs_flowControl", + "type": "ambiguous_selection", + "description": "How long users waited in queues should prefer dba_userDelay, not dba_flowControl", + "input": "How long were users waiting in the query queue between March 1 and March 31, 2024?", + "expected_tools": [ + { + "name": "dba_userDelay", + "params": { + "start_date": "2024-03-01", + "end_date": "2024-03-31" + } + } + ] + }, + { + "id": "dba_flowControl_ambiguous_vs_userDelay", + "type": "ambiguous_selection", + "description": "System throttling and flow-control statistics should prefer dba_flowControl, not dba_userDelay", + "input": "Show throttling and flow-control delay statistics from January 15 through February 28, 2024.", + "expected_tools": [ + { + "name": "dba_flowControl", + "params": { + "start_date": "2024-01-15", + "end_date": "2024-02-28" + } + } + ] + }, + { + "id": "dba_tableSpace_missing_db", + "type": "missing_parameter", + "description": "Agent must ask which database to list biggest tables for", + "input": "Show me the biggest tables", + "expected_tools": [] + }, + { + "id": "dba_tableSqlList_missing_table", + "type": "missing_parameter", + "description": "Agent must ask which table to list queries for", + "input": "Show me the queries from the last 14 days", + "expected_tools": [] + }, + { + "id": "dba_userSqlList_missing_user", + "type": "missing_parameter", + "description": "Agent must ask which user to list queries for", + "input": "Show me the queries a user executed over the past 14 days", + "expected_tools": [] + }, + { + "id": "dba_databaseSpace_missing_db", + "type": "missing_parameter", + "description": "Agent must ask which database to get storage for", + "input": "How much space is a specific database using?", + "expected_tools": [] + }, + { + "id": "dba_tableUsageImpact_missing_db", + "type": "missing_parameter", + "description": "Agent must ask which database to evaluate usage impact for", + "input": "Which users are hitting a particular database the most?", + "expected_tools": [] + }, + { + "id": "dba_databaseSpace_clarify_then_call", + "type": "missing_parameter", + "description": "Agent asks which database, then calls dba_databaseSpace after user clarifies", + "turns": [ + { + "input": "How much space is a specific database using?", + "expect": "clarification" + }, + { + "input": "How much disk space is the {EVALS_DATABASE} database using?", + "expected_tools": [ + { + "name": "dba_databaseSpace", + "params": { + "database_name": "{EVALS_DATABASE}" + } + } + ] + } + ] + }, + { + "id": "dba_tableSqlList_clarify_then_call", + "type": "missing_parameter", + "description": "Agent asks which table, then calls dba_tableSqlList after user clarifies", + "turns": [ + { + "input": "Show me the queries from the last 14 days", + "expect": "clarification" + }, + { + "input": "List SQL run against evals_orders over the past 14 days", + "expected_tools": [ + { + "name": "dba_tableSqlList", + "params": { + "table_name": "evals_orders", + "no_days": 14 + } + } + ] + } + ] + }, + { + "id": "dba_userSqlList_clarify_then_call", + "type": "missing_parameter", + "description": "Agent asks which user, then calls dba_userSqlList after user clarifies", + "turns": [ + { + "input": "Show me the queries a user executed over the past 14 days", + "expect": "clarification" + }, + { + "input": "List all SQL that user ANALYTICS_USER has run over the last 14 days", + "expected_tools": [ + { + "name": "dba_userSqlList", + "params": { + "user_name": "ANALYTICS_USER", + "no_days": 14 + } + } + ] + } + ] + }, + { + "id": "dba_unknown_tool_missing", + "type": "missing_parameter", + "description": "User asks for something no DBA tool supports — agent should explain and ask for clarification", + "input": "Automatically redistribute my table data across AMPs to improve skew", + "expected_tools": [] + } + ] +} diff --git a/tests/eval_test/cases/plot.json b/tests/eval_test/cases/plot.json new file mode 100644 index 0000000..1d8df39 --- /dev/null +++ b/tests/eval_test/cases/plot.json @@ -0,0 +1,138 @@ +{ + "module": "plot", + "cases": [ + { + "id": "plot_line_chart_happy", + "type": "happy_path", + "description": "Line chart of order amounts over time from evals_orders", + "input": "Show me a line chart of order amounts over time from {EVALS_DATABASE}.evals_orders using order_date on the x-axis and amount on the y-axis", + "expected_tools": [ + { + "name": "plot_line_chart", + "params": { + "table_name": "{EVALS_DATABASE}.evals_orders", + "labels": "order_date", + "columns": ["amount"] + } + } + ] + }, + { + "id": "plot_pie_chart_happy", + "type": "happy_path", + "description": "Pie chart breaking down orders by product category", + "input": "Plot {EVALS_DATABASE}.evals_orders as a pie chart using product_category for the slice labels and amount for the values.", + "expected_tools": [ + { + "name": "plot_pie_chart", + "params": { + "table_name": "{EVALS_DATABASE}.evals_orders", + "labels": "product_category", + "column": "amount" + } + } + ] + }, + { + "id": "plot_polar_chart_happy", + "type": "happy_path", + "description": "Polar area chart of order amounts by product category", + "input": "Create a polar area chart showing order amounts by product_category from {EVALS_DATABASE}.evals_orders", + "expected_tools": [ + { + "name": "plot_polar_chart", + "params": { + "table_name": "{EVALS_DATABASE}.evals_orders", + "labels": "product_category", + "column": "amount" + } + } + ] + }, + { + "id": "plot_radar_chart_happy", + "type": "happy_path", + "description": "Radar chart of salary by department in evals_employees", + "input": "Draw a spider chart from {EVALS_DATABASE}.evals_employees using department as the axis labels and salary as the values.", + "expected_tools": [ + { + "name": "plot_radar_chart", + "params": { + "table_name": "{EVALS_DATABASE}.evals_employees", + "labels": "department", + "columns": "salary" + } + } + ] + }, + { + "id": "plot_line_vs_radar_ambiguous", + "type": "ambiguous_selection", + "description": "Time-series data should use line chart, not radar", + "input": "Plot how order amounts change over time from {EVALS_DATABASE}.evals_orders using order_date on the x-axis", + "expected_tools": [ + { + "name": "plot_line_chart", + "params": { + "table_name": "{EVALS_DATABASE}.evals_orders", + "labels": "order_date", + "columns": ["amount"] + } + } + ] + }, + { + "id": "plot_pie_vs_polar_ambiguous", + "type": "ambiguous_selection", + "description": "Proportion/share breakdown should use pie chart over polar", + "input": "Use a pie chart to show {EVALS_DATABASE}.evals_orders with product_category as labels and amount as the values.", + "expected_tools": [ + { + "name": "plot_pie_chart", + "params": { + "table_name": "{EVALS_DATABASE}.evals_orders", + "labels": "product_category", + "column": "amount" + } + } + ] + }, + { + "id": "plot_missing_data", + "type": "missing_parameter", + "description": "Chart request with no table or data provided \u2014 agent must ask for clarification", + "input": "Create a chart for me", + "expected_tools": [] + }, + { + "id": "plot_pie_chart_direct", + "type": "happy_path", + "description": "Direct pie chart from table — no pre-query needed since plot tools read tables directly", + "input": "Give me a pie chart of amount by product_category from {EVALS_DATABASE}.evals_orders", + "expected_tools": [ + { + "name": "plot_pie_chart", + "params": { + "table_name": "{EVALS_DATABASE}.evals_orders", + "labels": "product_category", + "column": "amount" + } + } + ] + }, + { + "id": "plot_missing_table", + "type": "missing_parameter", + "description": "Agent must ask which table to plot", + "input": "Create a line chart of sales over time", + "expected_tools": [] + }, + { + "id": "plot_unknown_tool_missing", + "type": "missing_parameter", + "description": "User asks for a non\u2011existent plot tool", + "input": "Show me the network topology as a diagram", + "expected_tools": [] + } + ] +} diff --git a/tests/eval_test/cases/qlty.json b/tests/eval_test/cases/qlty.json new file mode 100644 index 0000000..b08c6ab --- /dev/null +++ b/tests/eval_test/cases/qlty.json @@ -0,0 +1,183 @@ +{ + "module": "qlty", + "cases": [ + { + "id": "qlty_missingValues_happy", + "type": "happy_path", + "description": "Find columns with nulls in evals_orders (ship_date is nullable)", + "input": "Which columns in {EVALS_DATABASE}.evals_orders contain null or empty entries?", + "expected_tools": [ + { + "name": "qlty_missingValues", + "params": {} + } + ] + }, + { + "id": "qlty_missingValues_ambiguous_vs_rowsWithMissingValues", + "type": "ambiguous_selection", + "description": "Asking which columns have nulls (column-level summary) should prefer qlty_missingValues over qlty_rowsWithMissingValues", + "input": "Which fields in {EVALS_DATABASE}.evals_orders have missing data?", + "expected_tools": [ + { + "name": "qlty_missingValues", + "params": {} + } + ] + }, + { + "id": "qlty_negativeValues_happy", + "type": "happy_path", + "description": "Find negative amounts in evals_orders (some amounts are negative refunds)", + "input": "Show me which fields in {EVALS_DATABASE}.evals_orders contain numbers below zero", + "expected_tools": [ + { + "name": "qlty_negativeValues", + "params": {} + } + ] + }, + { + "id": "qlty_distinctCategories_happy", + "type": "happy_path", + "description": "List unique product categories in evals_orders", + "input": "Show me all the unique values in the product_category column of {EVALS_DATABASE}.evals_orders", + "expected_tools": [ + { + "name": "qlty_distinctCategories", + "params": { + "column_name": "product_category" + } + } + ] + }, + { + "id": "qlty_distinctCategories_missing_column", + "type": "missing_parameter", + "description": "Agent must ask which column when none is specified", + "input": "What distinct values exist in a column of {EVALS_DATABASE}.evals_orders?", + "expected_tools": [] + }, + { + "id": "qlty_standardDeviation_happy", + "type": "happy_path", + "description": "Get mean and spread of salary in evals_employees", + "input": "What is the average and spread of salary values in {EVALS_DATABASE}.evals_employees?", + "expected_tools": [ + { + "name": "qlty_standardDeviation", + "params": { + "column_name": "salary" + } + } + ] + }, + { + "id": "qlty_standardDeviation_ambiguous_vs_univariateStatistics", + "type": "ambiguous_selection", + "description": "Asking specifically for standard deviation should prefer qlty_standardDeviation over the broader qlty_univariateStatistics", + "input": "What is the standard deviation of the salary field in {EVALS_DATABASE}.evals_employees?", + "expected_tools": [ + { + "name": "qlty_standardDeviation", + "params": { + "column_name": "salary" + } + } + ] + }, + { + "id": "qlty_columnSummary_happy", + "type": "happy_path", + "description": "Get a statistical summary of all columns in evals_employees", + "input": "Show me a statistical summary of every field in {EVALS_DATABASE}.evals_employees", + "expected_tools": [ + { + "name": "qlty_columnSummary", + "params": {} + } + ] + }, + { + "id": "qlty_univariateStatistics_happy", + "type": "happy_path", + "description": "Get full univariate stats for salary in evals_employees", + "input": "Give me a complete statistical breakdown of the salary field in {EVALS_DATABASE}.evals_employees", + "expected_tools": [ + { + "name": "qlty_univariateStatistics", + "params": { + "column_name": "salary" + } + } + ] + }, + { + "id": "qlty_univariateStatistics_ambiguous_vs_columnSummary", + "type": "ambiguous_selection", + "description": "Asking for stats on a specific column should prefer qlty_univariateStatistics over the table-level qlty_columnSummary", + "input": "Show me min, max, mean, and percentiles for the amount column in {EVALS_DATABASE}.evals_orders", + "expected_tools": [ + { + "name": "qlty_univariateStatistics", + "params": { + "column_name": "amount" + } + } + ] + }, + { + "id": "qlty_rowsWithMissingValues_happy", + "type": "happy_path", + "description": "Retrieve rows where ship_date is null in evals_orders", + "input": "Show me all records from {EVALS_DATABASE}.evals_orders where the ship_date is empty or null", + "expected_tools": [ + { + "name": "qlty_rowsWithMissingValues", + "params": { + "column_name": "ship_date" + } + } + ] + }, + { + "id": "qlty_rowsWithMissingValues_ambiguous_vs_missingValues", + "type": "ambiguous_selection", + "description": "Asking to retrieve the actual rows with missing data should prefer qlty_rowsWithMissingValues over qlty_missingValues", + "input": "Fetch the rows from {EVALS_DATABASE}.evals_orders that are missing a ship_date", + "expected_tools": [ + { + "name": "qlty_rowsWithMissingValues", + "params": { + "column_name": "ship_date" + } + } + ] + }, + { + "id": "qlty_multi_tool_summary_then_nulls", + "type": "multi_tool", + "description": "Get column summary then drill into rows with missing values", + "input": "First give me a column summary of {EVALS_DATABASE}.evals_orders, then show me the rows where ship_date is missing", + "expected_tools": [ + { + "name": "qlty_columnSummary", + "params": {} + }, + { + "name": "qlty_rowsWithMissingValues", + "params": { + "column_name": "ship_date" + } + } + ] + }, + { + "id": "qlty_unknown_tool_missing", + "type": "missing_parameter", + "description": "User asks for a non\u2011existent quality tool", + "input": "Give me a data quality score for the dataset", + "expected_tools": [] + } + ] +} diff --git a/tests/eval_test/cases/sec.json b/tests/eval_test/cases/sec.json new file mode 100644 index 0000000..d669b55 --- /dev/null +++ b/tests/eval_test/cases/sec.json @@ -0,0 +1,231 @@ +{ + "module": "sec", + "cases": [ + { + "id": "sec_userDbPermissions_ambiguous_vs_userRoles", + "type": "ambiguous_selection", + "description": "Asking about a user's access rights on databases should prefer sec_userDbPermissions, not sec_userRoles", + "input": "Which databases can the user 'bob' access and what can they do in each one?", + "expected_tools": [ + { + "name": "sec_userDbPermissions", + "params": { + "user_name": "bob" + } + } + ] + }, + { + "id": "sec_userDbPermissions_missing_user", + "type": "missing_parameter", + "description": "User asks about permissions without naming a user", + "input": "Show me the database permissions", + "expected_tools": [] + }, + { + "id": "sec_rolePermissions_ambiguous_vs_userDbPermissions", + "type": "ambiguous_selection", + "description": "Asking about a role's access should use sec_rolePermissions, not sec_userDbPermissions", + "input": "What access rights does the 'analyst' role have across all databases?", + "expected_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ] + }, + { + "id": "sec_userRoles_ambiguous_vs_rolePermissions", + "type": "ambiguous_selection", + "description": "Listing roles assigned to a user should prefer sec_userRoles, not sec_rolePermissions", + "input": "Which roles are currently assigned to user 'diana'?", + "expected_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "diana" + } + } + ] + }, + { + "id": "sec_rolePermissions_ambiguous_vs_userRoles", + "type": "ambiguous_selection", + "description": "Permissions granted to a named role should prefer sec_rolePermissions, not sec_userRoles", + "input": "What database access has the 'analyst' role been granted?", + "expected_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ] + }, + { + "id": "sec_userRoles_ambiguous_vs_userDbPermissions", + "type": "ambiguous_selection", + "description": "Asking about a user's role assignments should prefer sec_userRoles even though permissions is related", + "input": "Tell me which roles have been assigned to user 'diana'", + "expected_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "diana" + } + } + ] + }, + { + "id": "sec_multi_tool_user_roles_then_role_permissions", + "type": "multi_tool", + "description": "Find a user's roles, then inspect a specific named role's permissions in one pass", + "input": "Show me what roles 'alice' has, then list the permissions for the 'analyst' role", + "expected_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "alice" + } + }, + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ] + }, + { + "id": "sec_userDbPermissions_happy", + "type": "happy_path", + "description": "Show what databases a user can access and what they can do in each", + "input": "Show me what access rights jsmith has across all the different databases in the system.", + "expected_tools": [ + { + "name": "sec_userDbPermissions", + "params": { + "user_name": "jsmith" + } + } + ] + }, + { + "id": "sec_rolePermissions_happy", + "type": "happy_path", + "description": "Show what permissions a named role has", + "input": "What access rights are granted to the FinanceReporter role?", + "expected_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "FinanceReporter" + } + } + ] + }, + { + "id": "sec_userRoles_happy", + "type": "happy_path", + "description": "List the roles assigned to a user account", + "input": "Which roles are assigned to the database account jsmith?", + "expected_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "jsmith" + } + } + ] + }, + { + "id": "sec_rolePermissions_missing_role", + "type": "missing_parameter", + "description": "Agent must ask which role to inspect", + "input": "What access rights does the role have?", + "expected_tools": [] + }, + { + "id": "sec_userRoles_missing_user", + "type": "missing_parameter", + "description": "Agent must ask which user to retrieve roles for", + "input": "Show me the user roles", + "expected_tools": [] + }, + { + "id": "sec_userRoles_clarify_then_call", + "type": "missing_parameter", + "description": "Agent asks which user, then calls sec_userRoles after user clarifies", + "turns": [ + { + "input": "Show me the user roles", + "expect": "clarification" + }, + { + "input": "Which roles are assigned to user diana?", + "expected_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "diana" + } + } + ] + } + ] + }, + { + "id": "sec_rolePermissions_clarify_then_call", + "type": "missing_parameter", + "description": "Agent asks which role, then calls sec_rolePermissions after user clarifies", + "turns": [ + { + "input": "What access rights does the role have?", + "expect": "clarification" + }, + { + "input": "What database access has the analyst role been granted?", + "expected_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ] + } + ] + }, + { + "id": "sec_userDbPermissions_clarify_then_call", + "type": "missing_parameter", + "description": "Agent asks which user, then calls sec_userDbPermissions after user clarifies", + "turns": [ + { + "input": "Show me the database permissions", + "expect": "clarification" + }, + { + "input": "What databases can user bob access and what can they do in each?", + "expected_tools": [ + { + "name": "sec_userDbPermissions", + "params": { + "user_name": "bob" + } + } + ] + } + ] + }, + { + "id": "sec_unknown_tool_missing", + "type": "missing_parameter", + "description": "Vague access question with no user or role named — agent should ask for clarification", + "input": "What access does this user have in the system?", + "expected_tools": [] + } + ] +} diff --git a/tests/eval_test/cases/tmpl.json b/tests/eval_test/cases/tmpl.json new file mode 100644 index 0000000..a02ace3 --- /dev/null +++ b/tests/eval_test/cases/tmpl.json @@ -0,0 +1,9 @@ +{ + "module": "tmpl", + "cases": [ + { + "_comment": "Happy path cases are auto-generated by generate_cases.py — run it first, then add ambiguous_selection, missing_parameter, and multi_tool cases here by hand.", + "_instructions": "After running: uv run python generate_cases.py --module tmpl, edit this file to add edge case types." + } + ] +} diff --git a/tests/eval_test/description_overrides.json b/tests/eval_test/description_overrides.json new file mode 100644 index 0000000..44047bb --- /dev/null +++ b/tests/eval_test/description_overrides.json @@ -0,0 +1,4 @@ +{ + "_comment": "Dev-space description overrides. Gitignored. Edit freely, then promote to MCP server.", + "base_tableList": "Lists all tables and views within a specific Teradata database or schema. The database_name parameter is REQUIRED and must contain an explicit database name provided by the user \u2014 you must NOT call this tool with an empty string or without a database name. If the user asks to see tables but does not specify which database, ask them to clarify which database they want to list tables from before calling this tool. Use base_databaseList instead if the user is asking which databases exist on the system." +} diff --git a/tests/eval_test/docs/cases.md b/tests/eval_test/docs/cases.md new file mode 100644 index 0000000..fb55e92 --- /dev/null +++ b/tests/eval_test/docs/cases.md @@ -0,0 +1,97 @@ +# Cases & scoring + +## What case types test + +| Type | What it catches | +|---|---| +| `happy_path` | Agent picks the right tool with correct params | +| `ambiguous_selection` | Two tools could apply — descriptions must be distinct | +| `missing_parameter` | Agent must ask for clarification, not hallucinate | +| `multi_tool` | Chained tool calls in order | +| `multi_turn` | Shallow dialog — clarify first, then call the right tool (2–7 turns) | + +**Priority modules:** `base`, `dba`, `sec`, `qlty` · **Maintained:** `chat`, `plot`, `tmpl` + +Cases live in `cases/.json`. + +## Single-turn format + +```json +{ + "id": "base_readQuery_happy", + "type": "happy_path", + "description": "What this tests", + "input": "The natural language prompt sent to the agent", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { "sql": "SELECT * FROM {EVALS_DATABASE}.evals_employees SAMPLE 10" } + } + ] +} +``` + +- `missing_parameter`: set `expected_tools` to `[]` +- `multi_tool`: list expected tool calls in order +- Param names must match live MCP schemas (e.g. `sql` for `base_readQuery`) + +## Multi-turn format (optional) + +Use a `turns` array instead of top-level `input` / `expected_tools`. Scored in one MCP session. **2–7 turns.** + +```json +{ + "id": "base_tablePreview_clarify_then_call", + "type": "missing_parameter", + "description": "Agent asks which table, then previews after user clarifies", + "turns": [ + { "input": "Preview some rows for me", "expect": "clarification" }, + { + "input": "Preview rows from {EVALS_DATABASE}.evals_employees", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "{EVALS_DATABASE}", + "table_name": "evals_employees" + } + } + ] + } + ] +} +``` + +Each turn sets **exactly one** of `"expect": "clarification"` or non-empty `"expected_tools"`. IDs should contain `clarify_then_call` for `--type multi_turn` filtering. + +## Scoring + +Deterministic checks run first (`judge/checks.py`), then LLM judge metrics where needed. + +### Deterministic checks + +| Check | Applies to | +|---|---| +| No tool calls | `missing_parameter` and clarification turns | +| Exact tool name | Primary tool on happy/ambiguous; all tools on `multi_tool` | +| Exact param values | `database_name`, `table_name`, `column_name`, `user_name`, `role_name` | +| Param key presence | `sql` / `query` — key required; value judged by LLM | + +Structural failures fail immediately without invoking the judge. + +### LLM judge metrics + +| Metric | Applies to | +|---|---| +| `ToolCorrectnessMetric` | Cases with `expected_tools` | +| `Clarification Check` (GEval) | `missing_parameter` and clarification turns | + +## Adding cases for a new tool + +1. Add a `happy_path` case (or use [`backup/generate_cases.py`](../backup/README.md) to draft from live descriptions). +2. Add `ambiguous_selection` — overlapping tool pair, prompt that could call either, `expected_tools` set to the winner. +3. Add `missing_parameter` — vague prompt, `expected_tools: []`. +4. Optionally add `multi_turn` or `multi_tool` cases. +5. Optionally run [`backup/audit_cases.py`](../backup/README.md) for pair coverage. + +Use vocabulary **different from tool descriptions** — evals stress-test routing, not parrot descriptions. diff --git a/tests/eval_test/docs/setup.md b/tests/eval_test/docs/setup.md new file mode 100644 index 0000000..dc07fbd --- /dev/null +++ b/tests/eval_test/docs/setup.md @@ -0,0 +1,94 @@ +# Setup + +## Prerequisites + +- Python 3.11+ +- Teradata MCP Server running at `http://127.0.0.1:8001` connected to a ClearScape Analytics Experience instance +- AWS account with Bedrock access to an Anthropic Claude model + +## Install + +### Option A — uv (recommended) + +[uv](https://docs.astral.sh/uv/) manages the virtual environment automatically. + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh + +git clone +cd teradata-mcp-evals + +uv venv +uv sync +cp .env.example .env +``` + +With the venv active you can call scripts directly (`python run_evals.py`). Alternatively, prefix commands with `uv run`. + +### Option B — standard venv + pip + +```bash +git clone +cd teradata-mcp-evals + +python3 -m venv .venv +source .venv/bin/activate +pip install -e . +cp .env.example .env +``` + +## Environment variables + +Edit `.env`: + +```dotenv +AWS_REGION=us-east-1 +AWS_ACCESS_KEY_ID=your-access-key-id +AWS_SECRET_ACCESS_KEY=your-secret-access-key +BEDROCK_MODEL_ID=anthropic.claude-3-5-sonnet-20241022-v2:0 + +# Optional — defaults to BEDROCK_MODEL_ID +# BEDROCK_JUDGE_MODEL_ID=anthropic.claude-3-5-sonnet-20241022-v2:0 + +MCP_SERVER_URL=http://127.0.0.1:8001/mcp +EVALS_DATABASE=your_database_name + +AGENT_MAX_STEPS=5 +AGENT_MAX_STEPS_PER_TURN=3 + +# Optional — only when testing description overrides (see docs/workflow.md) +# USE_DESCRIPTION_OVERRIDES=1 +# DESCRIPTION_OVERRIDES_FILE=description_overrides.json +``` + +Credentials follow the standard boto3 chain — env vars, `~/.aws/credentials`, or an IAM instance profile. Set `AWS_SESSION_TOKEN` for temporary credentials. + +## Test data + +Create eval tables once before running live evals: + +```bash +python setup_test_data.py +``` + +| Table | Purpose | +|---|---| +| `evals_employees` | Used by base, qlty, plot cases | +| `evals_orders` | Used by base, qlty, plot, dba cases (includes nullable `ship_date`, negative `amount`) | + +Set `EVALS_DATABASE` to the database where tables are created — usually your ClearScape username. Case JSON uses `{EVALS_DATABASE}` as a runtime placeholder. + +```bash +python teardown_test_data.py # clean up +python setup_test_data.py --drop-first # recreate from scratch +python preflight.py # verify tables exist (also run by run_evals.py) +``` + +## Unit tests + +No MCP server or Bedrock required: + +```bash +uv run pytest tests/test_checks.py tests/test_multi_turn.py tests/test_report.py \ + tests/test_suggest_overrides.py tests/test_description_overrides.py -v +``` diff --git a/tests/eval_test/docs/structure.md b/tests/eval_test/docs/structure.md new file mode 100644 index 0000000..c49216c --- /dev/null +++ b/tests/eval_test/docs/structure.md @@ -0,0 +1,35 @@ +# Project structure + +``` +teradata-mcp-evals/ + agent/ + client.py # MCP agent; optional description overrides (opt-in) + judge/ + bedrock_llm.py # Bedrock wrapper for deepeval judge + checks.py # Deterministic structural checks + metrics.py # ToolCorrectnessMetric + clarification GEval + report.py # Eval summaries → results/ + suggest_overrides.py # LLM draft overrides (library) + cases/ + *.json # Eval cases per MCP module prefix + tests/ + conftest.py # Fixtures, {EVALS_DATABASE} substitution + case_runner.py # Case execution and scoring + test_.py # One pytest file per module + test_checks.py # Unit tests — deterministic checks + test_multi_turn.py # Unit tests — multi-turn schema + test_report.py # Unit tests — eval summaries + test_suggest_overrides.py + test_description_overrides.py + docs/ # Extended documentation + backup/ # Optional bootstrap/audit scripts + run_evals.py # CLI entry point + suggest_overrides.py # CLI — draft description overrides + setup_test_data.py + preflight.py + teardown_test_data.py + pyproject.toml + .env.example +``` + +Optional scripts in [`backup/`](../backup/README.md) are not required for the main eval workflow. diff --git a/tests/eval_test/docs/workflow.md b/tests/eval_test/docs/workflow.md new file mode 100644 index 0000000..e452763 --- /dev/null +++ b/tests/eval_test/docs/workflow.md @@ -0,0 +1,133 @@ +# Running evals & description workflow + +## End-to-end flow + +The main loop measures **live MCP tool descriptions**, improves wording locally, then promotes changes back to the MCP server. + +``` +setup_test_data.py → run_evals.py (baseline) + ↓ + results/runs//summary.{md,json} + results/latest.json ← pointer to latest run + ↓ + suggest_overrides.py + ↓ + results/runs//suggested_overrides.json + ↓ + suggest_overrides.py --apply + ↓ + description_overrides.json + ↓ + run_evals.py --with-description-overrides + ↓ + promote to MCP server → run_evals.py (baseline again) +``` + +| Step | Command / file | Purpose | +|------|----------------|---------| +| 1. Prepare | `setup_test_data.py`, MCP server running | Eval tables + live tools | +| 2. Baseline | `run_evals.py` | Agent sees **live MCP descriptions**; writes artifacts under `results/runs//` | +| 3. Triage | `results/latest_summary.md` or run directory | Pass/fail counts, eval prompts, recommendations | +| 4. Draft fixes | `suggest_overrides.py` | Bedrock drafts revised descriptions for failed `ambiguous_selection` cases | +| 5. Review | run's `suggested_overrides.json` | Human review — **not applied automatically** | +| 6. Apply | `uv run python suggest_overrides.py --apply` | Replace `description_overrides.json` with draft suggestions only | +| 6. Test locally | `description_overrides.json` + `--with-description-overrides` | Patch descriptions without changing the MCP server | +| 7. Ship | Edit descriptions in MCP server repo | Re-run baseline evals to confirm | + +## Running evals + +Before a live run: test tables created, MCP server at `MCP_SERVER_URL`. `run_evals.py` runs preflight automatically. + +```bash +python run_evals.py # all modules +python run_evals.py --module base # one module +python run_evals.py --module base --type ambiguous_selection +python run_evals.py --verbose +python run_evals.py --run-label after-tablelist-fix # optional label in run directory name +python run_evals.py --list-runs # show recent runs +``` + +### `--type` filter + +| Value | Matches case IDs containing | +|---|---| +| `happy_path` | `happy` | +| `ambiguous_selection` | `ambiguous` | +| `missing_parameter` | `missing` | +| `multi_tool` | `multi_tool` | +| `multi_turn` | `clarify_then_call` | + +Advanced filtering via deepeval/pytest, e.g. `deepeval test run tests/ -k "test_base and tablePreview"`. + +## Baseline vs overrides + +**Default:** evals use live MCP server tool descriptions. + +**Opt-in overrides:** review the draft, then apply with `suggest_overrides.py --apply`, or re-run with: + +```bash +python run_evals.py --module base --with-description-overrides +python run_evals.py --module base --description-overrides-file my_overrides.json +``` + +Or set `USE_DESCRIPTION_OVERRIDES=1` / `DESCRIPTION_OVERRIDES_FILE=...` in `.env`. + +### Suggesting override drafts + +After a baseline run with failures: + +```bash +uv run python suggest_overrides.py +uv run python suggest_overrides.py --type missing_parameter +uv run python suggest_overrides.py --type ambiguous_selection,happy_path +``` + +Processes failed **`ambiguous_selection`**, **`happy_path`**, **`missing_parameter`**, and **`multi_tool`** cases by default. Each type gets a tailored prompt: + +| Case type | What it revises | +|---|---| +| `ambiguous_selection` | Both competing tools — sharpen routing boundaries | +| `happy_path` | Expected tool (and wrong tool if one was called) | +| `missing_parameter` | Tools called prematurely — add “ask first” constraints | +| `multi_tool` | Tools in the workflow — clarify sequencing | + +`missing_parameter` cases where the agent never called a tool (judge-only clarification failure) are **skipped** — those are agent behavior, not description routing. + +Flags: `--summary`, `--output`, `--overrides-file`, `--type`, `--dry-run`. + +Reads the latest run summary (`results/latest.json` → `results/runs//summary.json`), fetches live MCP descriptions, and writes the draft into that run directory (also copied to `results/suggested_overrides.json`). Review the draft, then apply: + +```bash +uv run python suggest_overrides.py --apply +uv run python suggest_overrides.py --apply --dry-run # preview merge +uv run python suggest_overrides.py --apply --tools base_tableList # one tool only +uv run python run_evals.py --with-description-overrides +``` + +## Results layout + +Each eval run gets its own directory under `results/runs/`. The directory name encodes when and how the run was executed, for example: + +`2026-06-23T20-40-34.611558Z__base__overrides` + +Format: `____[__][__]` + +| Path | Contents | +|---|---| +| `results/runs//summary.md` | Human-readable triage report for that run | +| `results/runs//summary.json` | Structured case results | +| `results/runs//manifest.json` | Run metadata and artifact list | +| `results/runs//suggested_overrides.json` | LLM draft from that run's failures (when generated) | +| `results/latest.json` | Pointer to the most recent run | +| `results/index.json` | Newest-first list of recent runs with pass/fail counts | +| `results/latest_summary.md` | Copy of the latest run summary (backward compatible) | +| `results/latest_summary.json` | Copy of the latest run JSON (backward compatible) | +| `results/suggested_overrides.json` | Copy of the latest suggestion draft (backward compatible) | + +List recent runs: + +```bash +uv run python run_evals.py --list-runs +``` + +Other files may appear under `results/` from optional tooling (audit logs, live description dumps, etc.) — those are not tied to a specific eval run. diff --git a/tests/eval_test/judge/__init__.py b/tests/eval_test/judge/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/eval_test/judge/bedrock_llm.py b/tests/eval_test/judge/bedrock_llm.py new file mode 100644 index 0000000..50ba4e1 --- /dev/null +++ b/tests/eval_test/judge/bedrock_llm.py @@ -0,0 +1,57 @@ +"""DeepEval LLM wrapper backed by AWS Bedrock (Converse API).""" + +from __future__ import annotations + +import json +import os +from typing import Any + +import boto3 +from deepeval.models.base_model import DeepEvalBaseLLM + + +class BedrockLLM(DeepEvalBaseLLM): + def __init__(self, model_id: str | None = None, bedrock_client=None): + self.model_id = model_id or os.environ.get( + "BEDROCK_JUDGE_MODEL_ID", + os.environ.get("BEDROCK_MODEL_ID", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ) + self._client = bedrock_client or self._build_client() + super().__init__() + + def _build_client(self): + region = os.environ.get("AWS_REGION", "us-east-1") + return boto3.client("bedrock-runtime", region_name=region) + + def load_model(self): + return self._client + + def get_model_name(self) -> str: + return self.model_id + + def _call(self, prompt: str, schema: Any = None) -> str: + response = self._client.converse( + modelId=self.model_id, + messages=[{"role": "user", "content": [{"text": prompt}]}], + ) + text = response["output"]["message"]["content"][0]["text"] + + if schema is not None: + # deepeval passes a Pydantic model as schema for structured output; + # extract the JSON block from the response and parse it. + try: + start = text.index("{") + end = text.rindex("}") + 1 + return schema.model_validate(json.loads(text[start:end])) + except Exception: + pass + + return text + + def generate(self, prompt: str, schema: Any = None) -> str: + return self._call(prompt, schema) + + async def a_generate(self, prompt: str, schema: Any = None) -> str: + import asyncio + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._call, prompt, schema) diff --git a/tests/eval_test/judge/checks.py b/tests/eval_test/judge/checks.py new file mode 100644 index 0000000..59875ad --- /dev/null +++ b/tests/eval_test/judge/checks.py @@ -0,0 +1,114 @@ +"""Deterministic structural checks that run before the LLM judge.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +# Param values compared exactly when non-empty in the expected case. +EXACT_VALUE_KEYS = frozenset( + { + "database_name", + "table_name", + "column_name", + "user_name", + "username", + "role_name", + } +) + +# Param keys that must be present but may differ in value (e.g. SQL wording). +PRESENCE_ONLY_KEYS = frozenset({"sql", "query"}) + + +@dataclass +class ToolCallRecord: + name: str + input_parameters: dict[str, Any] + + +def _check_params(expected_params: dict[str, Any], actual_params: dict[str, Any]) -> list[str]: + errors: list[str] = [] + + for key, expected_value in expected_params.items(): + if key not in actual_params: + errors.append(f"missing param key '{key}'") + continue + + actual_value = actual_params[key] + + if key in PRESENCE_ONLY_KEYS: + if expected_value and not actual_value: + errors.append(f"param '{key}' must not be empty") + continue + + if key in EXACT_VALUE_KEYS and expected_value not in ("", None): + if actual_value != expected_value: + errors.append( + f"param '{key}': expected {expected_value!r}, got {actual_value!r}", + ) + + return errors + + +def _check_tool_pair( + expected: ToolCallRecord, + actual: ToolCallRecord, + *, + label: str, +) -> list[str]: + errors: list[str] = [] + if actual.name != expected.name: + errors.append(f"{label}: expected tool {expected.name!r}, got {actual.name!r}") + errors.extend(_check_params(expected.input_parameters, actual.input_parameters)) + return errors + + +def run_deterministic_checks( + case: dict, + tools_called: list[ToolCallRecord], +) -> list[str]: + """Return a list of structural check failures (empty list means pass).""" + case_type = case.get("type", "happy_path") + expected_raw = case.get("expected_tools", []) + expected = [ + ToolCallRecord(name=t["name"], input_parameters=t.get("params", {})) + for t in expected_raw + ] + + if case_type == "missing_parameter": + if tools_called: + names = [tc.name for tc in tools_called] + return [f"expected no tool calls for missing_parameter case, got {names}"] + return [] + + if not expected: + return [] + + errors: list[str] = [] + + if case_type == "multi_tool": + if len(tools_called) != len(expected): + errors.append( + f"multi_tool: expected {len(expected)} tool call(s), got {len(tools_called)}", + ) + return errors + for i, (exp, act) in enumerate(zip(expected, tools_called, strict=True)): + errors.extend(_check_tool_pair(exp, act, label=f"step {i + 1}")) + return errors + + if not tools_called: + return ["expected at least one tool call, got none"] + + errors.extend(_check_tool_pair(expected[0], tools_called[0], label="primary tool")) + + return errors + + +def assert_deterministic_checks(case: dict, tools_called: list[ToolCallRecord]) -> None: + """Fail fast on structural mismatch before invoking the LLM judge.""" + errors = run_deterministic_checks(case, tools_called) + if errors: + case_id = case.get("id", "") + detail = "; ".join(errors) + raise AssertionError(f"[{case_id}] deterministic check failed: {detail}") diff --git a/tests/eval_test/judge/metrics.py b/tests/eval_test/judge/metrics.py new file mode 100644 index 0000000..3c7fa2b --- /dev/null +++ b/tests/eval_test/judge/metrics.py @@ -0,0 +1,43 @@ +"""deepeval metrics for the Teradata MCP eval suite.""" + +from __future__ import annotations + +from deepeval.metrics import GEval, ToolCorrectnessMetric +from deepeval.test_case import LLMTestCaseParams + + +def tool_correctness_metric(judge_llm) -> ToolCorrectnessMetric: + """Evaluates tool selection accuracy and parameter correctness via LLM judge.""" + return ToolCorrectnessMetric( + threshold=0.5, + model=judge_llm, + include_reason=True, + ) + + +def clarification_metric(judge_llm) -> GEval: + """For missing_parameter cases: checks the agent asked for clarification rather than hallucinating.""" + return GEval( + name="Clarification Check", + criteria=( + "The agent was given a prompt that is missing a required parameter. " + "The agent should ask the user for the missing information rather than " + "inventing a value or proceeding with a guess. " + "Score 1.0 if the response contains a clear request for the missing information. " + "Score 0.0 if the agent fabricated a parameter value or called a tool without asking." + ), + evaluation_params=[ + LLMTestCaseParams.INPUT, + LLMTestCaseParams.ACTUAL_OUTPUT, + ], + model=judge_llm, + threshold=0.5, + ) + + +def get_metrics(case: dict, judge_llm) -> list: + """Return the appropriate metric set for a given test case type.""" + metrics = [tool_correctness_metric(judge_llm)] + if case.get("type") == "missing_parameter": + metrics.append(clarification_metric(judge_llm)) + return metrics diff --git a/tests/eval_test/judge/report.py b/tests/eval_test/judge/report.py new file mode 100644 index 0000000..9b134cd --- /dev/null +++ b/tests/eval_test/judge/report.py @@ -0,0 +1,561 @@ +"""Collect eval outcomes and write human-readable summaries to results/.""" + +from __future__ import annotations + +import json +import os +import re +from dataclasses import asdict, dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +RESULTS_DIR = Path(__file__).resolve().parent.parent / "results" +RUNS_DIR = RESULTS_DIR / "runs" +LATEST_POINTER_FILE = RESULTS_DIR / "latest.json" +INDEX_FILE = RESULTS_DIR / "index.json" +MAX_INDEX_RUNS = 50 + + +@dataclass +class CaseEvalResult: + case_id: str + case_type: str + description: str + input: str + expected_tools: list[dict[str, Any]] + passed: bool + failure_stage: str | None = None + failure_detail: str | None = None + actual_tools: list[dict[str, Any]] | None = None + actual_output: str | None = None + metric_reasons: list[str] = field(default_factory=list) + recommendation: str | None = None + turn_details: list[dict[str, Any]] | None = None + + +@dataclass +class EvalRunReport: + started_at: str + module_filter: str + case_type_filter: str + agent_model_id: str + judge_model_id: str + evals_database: str + description_mode: str = "mcp_server" + description_overrides_file: str | None = None + description_override_count: int = 0 + run_label: str | None = None + results: list[CaseEvalResult] = field(default_factory=list) + + @property + def total(self) -> int: + return len(self.results) + + @property + def passed_count(self) -> int: + return sum(1 for result in self.results if result.passed) + + @property + def failed_count(self) -> int: + return self.total - self.passed_count + + +_report: EvalRunReport | None = None + + +def begin_eval_run( + *, + agent_model_id: str, + judge_model_id: str, + evals_database: str, + description_mode: str = "mcp_server", + description_overrides_file: str | None = None, + description_override_count: int = 0, + run_label: str | None = None, +) -> None: + """Start a fresh in-memory report for the current pytest session.""" + global _report + _report = EvalRunReport( + started_at=datetime.now(timezone.utc).isoformat(), + module_filter=os.environ.get("EVALS_RUN_MODULE", "all"), + case_type_filter=os.environ.get("EVALS_RUN_TYPE", "all"), + agent_model_id=agent_model_id, + judge_model_id=judge_model_id, + evals_database=evals_database, + description_mode=description_mode, + description_overrides_file=description_overrides_file, + description_override_count=description_override_count, + run_label=run_label or os.environ.get("EVALS_RUN_LABEL") or None, + ) + + +def record_case_result(result: CaseEvalResult) -> None: + """Append a case outcome to the current report.""" + if _report is None: + return + _report.results.append(result) + + +def get_current_report() -> EvalRunReport | None: + return _report + + +def _tool_names(tools: list[dict[str, Any]] | None) -> list[str]: + if not tools: + return [] + return [tool["name"] for tool in tools] + + +def _competing_tool_hint(description: str, expected_tool: str) -> str | None: + patterns = [ + rf"\bnot\s+({expected_tool.split('_', 1)[0]}_\w+)\b", + rf"\bover\s+({expected_tool.split('_', 1)[0]}_\w+)\b", + rf"\bvs\.?\s+({expected_tool.split('_', 1)[0]}_\w+)\b", + rf"\binstead of\s+({expected_tool.split('_', 1)[0]}_\w+)\b", + ] + for pattern in patterns: + match = re.search(pattern, description, flags=re.IGNORECASE) + if match: + return match.group(1) + generic = re.search(r"\b(base|dba|sec|qlty|chat|plot|tmpl)_\w+\b", description) + if generic and generic.group(0) != expected_tool: + return generic.group(0) + return None + + +def build_recommendation( + case: dict, + *, + failure_stage: str, + failure_detail: str | None, + expected_tools: list[dict[str, Any]], + actual_tools: list[dict[str, Any]] | None, + metric_reasons: list[str], +) -> str: + """Turn a failure into actionable guidance on prompts and tool descriptions.""" + case_type = case.get("type", "happy_path") + description = case.get("description", "") + prompt = case.get("input") or "(multi-turn — see turn details)" + expected_names = _tool_names(expected_tools) + actual_names = _tool_names(actual_tools) + expected_primary = expected_names[0] if expected_names else "the expected tool" + actual_primary = actual_names[0] if actual_names else "no tool" + + if failure_stage == "agent": + return ( + f"The agent loop failed before scoring: {failure_detail}. " + "Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable " + f"with the current environment (`EVALS_DATABASE={os.environ.get('EVALS_DATABASE', '')}`)." + ) + + override_note = "" + if os.environ.get("USE_DESCRIPTION_OVERRIDES", "").lower() in {"1", "true", "yes"} or os.environ.get( + "DESCRIPTION_OVERRIDES_FILE" + ): + override_note = ( + " This run used description_overrides.json rather than live MCP descriptions — " + "if a revised override fixes routing, promote that wording to the MCP server." + ) + + if case_type == "ambiguous_selection": + competing = _competing_tool_hint(description, expected_primary) + competitor_text = ( + f" and `{competing}`" if competing and competing != expected_primary else "" + ) + return ( + f"This case tests routing between `{expected_primary}`{competitor_text}. " + f"Eval prompt: \"{prompt}\". " + f"The agent chose `{actual_primary}` instead of `{expected_primary}`. " + "Recommendation: tighten the MCP tool descriptions so only one tool clearly applies — " + f"sharpen `{expected_primary}` for this scenario or narrow `{actual_primary}` so it does not absorb it. " + "Also confirm the eval prompt uses vocabulary different from both descriptions and is not biased " + f"toward `{actual_primary}`." + f"{override_note}" + ) + + if case_type == "missing_parameter": + if actual_names: + return ( + f"The prompt deliberately omits required information: \"{prompt}\". " + f"The agent called `{', '.join(actual_names)}` instead of asking for clarification. " + "Recommendation: review whether those tool descriptions invite calls without full context, " + "and keep the eval prompt vague enough that asking is the only reasonable response." + ) + return ( + f"The prompt deliberately omits required information: \"{prompt}\". " + "The agent should ask for the missing detail rather than guess or call a tool. " + f"Judge feedback: {'; '.join(metric_reasons) or failure_detail or 'did not request clarification'}. " + "Recommendation: make the missing field obvious in the agent system prompt, or tighten tool descriptions " + "that encourage premature tool use." + ) + + if case_type == "multi_tool": + return ( + f"This workflow case expects tools in order: {', '.join(expected_names)}. " + f"The agent produced: {', '.join(actual_names) or 'no tool calls'}. " + f"Eval prompt: \"{prompt}\". " + f"Failure detail: {failure_detail or '; '.join(metric_reasons)}. " + "Recommendation: verify each tool description mentions when it belongs in a multi-step flow, " + "and confirm the eval prompt states the full task without naming tools directly." + ) + + return ( + f"Expected `{expected_primary}` for prompt: \"{prompt}\". " + f"The agent produced `{actual_primary}`. " + f"Failure detail: {failure_detail or '; '.join(metric_reasons) or 'metric check failed'}. " + "Recommendation: compare the eval prompt with the MCP description for " + f"`{expected_primary}` — the prompt should map clearly to that tool without copying description wording." + f"{override_note}" + ) + + +def _format_tools(tools: list[dict[str, Any]] | None) -> str: + if not tools: + return "_none_" + lines = [] + for tool in tools: + params = json.dumps(tool.get("params", {}), indent=2) + lines.append(f"- `{tool['name']}` with params:\n```json\n{params}\n```") + return "\n".join(lines) + + +def render_markdown(report: EvalRunReport) -> str: + """Render a markdown summary for the eval run.""" + lines = [ + "# Teradata MCP Eval Run Summary", + "", + f"**Started (UTC):** {report.started_at}", + f"**Module filter:** {report.module_filter}", + f"**Case type filter:** {report.case_type_filter}", + f"**Agent model:** {report.agent_model_id}", + f"**Judge model:** {report.judge_model_id}", + f"**Eval database:** {report.evals_database or '(not set)'}", + ] + if report.run_label: + lines.append(f"**Run label:** {report.run_label}") + if report.description_mode == "overrides": + lines.append( + f"**Tool descriptions:** overrides from `{report.description_overrides_file or 'description_overrides.json'}` " + f"({report.description_override_count} tool(s) patched)" + ) + else: + lines.append("**Tool descriptions:** live MCP server (baseline)") + lines.extend( + [ + "", + "## Overview", + "", + "| Metric | Count |", + "| --- | ---: |", + f"| Total cases | {report.total} |", + f"| Passed | {report.passed_count} |", + f"| Failed | {report.failed_count} |", + "", + ] + ) + + failed = [result for result in report.results if not result.passed] + passed = [result for result in report.results if result.passed] + + if failed: + lines.extend(["## Failed cases", ""]) + for result in failed: + lines.extend( + [ + f"### {result.case_id} ({result.case_type})", + "", + f"**Description:** {result.description or '—'}", + "", + "**Eval prompt:**", + "", + f"> {result.input}", + "", + "**Expected tool(s):**", + "", + _format_tools(result.expected_tools), + "", + "**Actual tool(s):**", + "", + _format_tools(result.actual_tools), + "", + ] + ) + if result.actual_output: + lines.extend(["**Agent response (excerpt):**", "", f"> {result.actual_output[:500]}", ""]) + if result.failure_stage or result.failure_detail: + lines.append( + f"**Failure ({result.failure_stage or 'unknown'}):** " + f"{result.failure_detail or '; '.join(result.metric_reasons) or '—'}" + ) + lines.append("") + if result.metric_reasons: + lines.append("**Judge notes:**") + lines.append("") + for reason in result.metric_reasons: + lines.append(f"- {reason}") + lines.append("") + if result.turn_details: + lines.extend(["**Turn details:**", "", "```json", json.dumps(result.turn_details, indent=2), "```", ""]) + if result.recommendation: + lines.extend(["**Recommendation:**", "", result.recommendation, ""]) + else: + lines.extend(["## Failed cases", "", "_None — all cases passed._", ""]) + + lines.extend(["## Passed cases", ""]) + if passed: + for result in passed: + lines.append(f"- `{result.case_id}` ({result.case_type})") + else: + lines.append("_None._") + + lines.append("") + return "\n".join(lines) + + +def _slugify(text: str) -> str: + slug = re.sub(r"[^\w-]+", "-", text.strip().lower()).strip("-") + return slug[:40] if slug else "run" + + +def build_run_id(report: EvalRunReport) -> str: + """Build a human-readable directory name for an eval run.""" + started_at = report.started_at + if started_at.endswith("+00:00"): + started_at = f"{started_at[:-6]}Z" + timestamp = started_at.replace(":", "-") + module = _slugify(report.module_filter or "all") + mode = "overrides" if report.description_mode == "overrides" else "baseline" + parts = [timestamp, module, mode] + if report.case_type_filter and report.case_type_filter != "all": + parts.append(_slugify(report.case_type_filter)) + if report.run_label: + parts.append(_slugify(report.run_label)) + return "__".join(parts) + + +@dataclass +class RunArtifacts: + run_id: str + run_dir: Path + summary_md: Path + summary_json: Path + manifest: Path + + +def load_latest_pointer() -> dict[str, Any] | None: + """Return the latest run pointer document, if present.""" + if not LATEST_POINTER_FILE.exists(): + return None + try: + data = json.loads(LATEST_POINTER_FILE.read_text(encoding="utf-8")) + except Exception: + return None + return data if isinstance(data, dict) else None + + +def resolve_default_summary_path() -> Path: + """Prefer the latest run's summary.json; fall back to legacy latest_summary.json.""" + pointer = load_latest_pointer() + if pointer: + summary_json = pointer.get("summary_json") + if isinstance(summary_json, str): + candidate = RESULTS_DIR / summary_json + if candidate.exists(): + return candidate + legacy = RESULTS_DIR / "latest_summary.json" + if legacy.exists(): + return legacy + return legacy + + +def run_dir_for_summary_path(summary_path: Path) -> Path | None: + """Return the run directory that owns a summary file, when known.""" + resolved = summary_path.resolve() + try: + if resolved.parent.parent == RUNS_DIR.resolve() and resolved.name in {"summary.json", "summary.md"}: + return resolved.parent + except Exception: + pass + + pointer = load_latest_pointer() + if pointer and summary_path.name == "latest_summary.json": + run_dir = pointer.get("run_dir") + if isinstance(run_dir, str): + return RESULTS_DIR / run_dir + return None + + +def resolve_suggestion_output_path(summary_path: Path) -> Path: + """Write suggestion drafts alongside the eval run when possible.""" + run_dir = run_dir_for_summary_path(summary_path) + if run_dir is not None: + return run_dir / "suggested_overrides.json" + return RESULTS_DIR / "suggested_overrides.json" + + +def _summary_payload(report: EvalRunReport) -> dict[str, Any]: + return { + "run_id": build_run_id(report), + "started_at": report.started_at, + "module_filter": report.module_filter, + "case_type_filter": report.case_type_filter, + "run_label": report.run_label, + "agent_model_id": report.agent_model_id, + "judge_model_id": report.judge_model_id, + "evals_database": report.evals_database, + "description_mode": report.description_mode, + "description_overrides_file": report.description_overrides_file, + "description_override_count": report.description_override_count, + "total": report.total, + "passed": report.passed_count, + "failed": report.failed_count, + "cases": [asdict(result) for result in report.results], + } + + +def _manifest_payload(report: EvalRunReport, *, run_id: str) -> dict[str, Any]: + return { + "run_id": run_id, + "started_at": report.started_at, + "module_filter": report.module_filter, + "case_type_filter": report.case_type_filter, + "run_label": report.run_label, + "description_mode": report.description_mode, + "description_overrides_file": report.description_overrides_file, + "description_override_count": report.description_override_count, + "agent_model_id": report.agent_model_id, + "judge_model_id": report.judge_model_id, + "evals_database": report.evals_database, + "total": report.total, + "passed": report.passed_count, + "failed": report.failed_count, + "artifacts": { + "summary_md": "summary.md", + "summary_json": "summary.json", + "manifest": "manifest.json", + "suggested_overrides": "suggested_overrides.json", + }, + } + + +def _index_entry(report: EvalRunReport, *, run_id: str) -> dict[str, Any]: + return { + "run_id": run_id, + "started_at": report.started_at, + "module_filter": report.module_filter, + "case_type_filter": report.case_type_filter, + "run_label": report.run_label, + "description_mode": report.description_mode, + "passed": report.passed_count, + "failed": report.failed_count, + "total": report.total, + "run_dir": f"runs/{run_id}", + "summary_json": f"runs/{run_id}/summary.json", + "summary_md": f"runs/{run_id}/summary.md", + } + + +def _update_run_index(entry: dict[str, Any]) -> None: + existing: list[dict[str, Any]] = [] + if INDEX_FILE.exists(): + try: + data = json.loads(INDEX_FILE.read_text(encoding="utf-8")) + if isinstance(data, dict) and isinstance(data.get("runs"), list): + existing = [item for item in data["runs"] if isinstance(item, dict)] + except Exception: + existing = [] + + run_id = entry.get("run_id") + existing = [item for item in existing if item.get("run_id") != run_id] + runs = [entry, *existing][:MAX_INDEX_RUNS] + INDEX_FILE.write_text(json.dumps({"runs": runs}, indent=2) + "\n", encoding="utf-8") + + +def load_run_index() -> list[dict[str, Any]]: + """Return indexed eval runs, newest first.""" + if not INDEX_FILE.exists(): + return [] + try: + data = json.loads(INDEX_FILE.read_text(encoding="utf-8")) + if isinstance(data, dict) and isinstance(data.get("runs"), list): + return [item for item in data["runs"] if isinstance(item, dict)] + except Exception: + pass + return [] + + +def format_run_index(limit: int = 20) -> str: + """Render a compact table of recent eval runs.""" + runs = load_run_index()[:limit] + if not runs: + return "No indexed eval runs yet." + + lines = [ + "Recent eval runs (newest first):", + "", + f"{'Run ID':<56} {'Mode':<10} {'Pass':>4} {'Fail':>4}", + f"{'-' * 56} {'-' * 10} {'-' * 4} {'-' * 4}", + ] + for run in runs: + run_id = str(run.get("run_id", "?"))[:56] + mode = str(run.get("description_mode", "?"))[:10] + passed = run.get("passed", "?") + failed = run.get("failed", "?") + lines.append(f"{run_id:<56} {mode:<10} {passed:>4} {failed:>4}") + lines.extend(["", f"Latest pointer: {LATEST_POINTER_FILE}", f"Full index: {INDEX_FILE}"]) + return "\n".join(lines) + + +def write_eval_summary(report: EvalRunReport) -> RunArtifacts: + """Write run artifacts under results/runs// and update index pointers.""" + RESULTS_DIR.mkdir(parents=True, exist_ok=True) + RUNS_DIR.mkdir(parents=True, exist_ok=True) + + run_id = build_run_id(report) + run_dir = RUNS_DIR / run_id + run_dir.mkdir(parents=True, exist_ok=True) + + summary_md = run_dir / "summary.md" + summary_json = run_dir / "summary.json" + manifest = run_dir / "manifest.json" + latest_markdown = RESULTS_DIR / "latest_summary.md" + latest_json = RESULTS_DIR / "latest_summary.json" + + markdown = render_markdown(report) + payload = _summary_payload(report) + manifest_payload = _manifest_payload(report, run_id=run_id) + + summary_md.write_text(markdown + "\n", encoding="utf-8") + summary_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + manifest.write_text(json.dumps(manifest_payload, indent=2) + "\n", encoding="utf-8") + + latest_markdown.write_text(markdown + "\n", encoding="utf-8") + latest_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + run_dir_rel = f"runs/{run_id}" + pointer = { + "run_id": run_id, + "started_at": report.started_at, + "run_dir": run_dir_rel, + "summary_md": f"{run_dir_rel}/summary.md", + "summary_json": f"{run_dir_rel}/summary.json", + "manifest": f"{run_dir_rel}/manifest.json", + "module_filter": report.module_filter, + "case_type_filter": report.case_type_filter, + "run_label": report.run_label, + "description_mode": report.description_mode, + "passed": report.passed_count, + "failed": report.failed_count, + "total": report.total, + } + LATEST_POINTER_FILE.write_text(json.dumps(pointer, indent=2) + "\n", encoding="utf-8") + _update_run_index(_index_entry(report, run_id=run_id)) + + return RunArtifacts( + run_id=run_id, + run_dir=run_dir, + summary_md=summary_md, + summary_json=summary_json, + manifest=manifest, + ) diff --git a/tests/eval_test/judge/suggest_overrides.py b/tests/eval_test/judge/suggest_overrides.py new file mode 100644 index 0000000..cc48aa3 --- /dev/null +++ b/tests/eval_test/judge/suggest_overrides.py @@ -0,0 +1,657 @@ +"""LLM-assisted draft descriptions for description_overrides.json from eval failures.""" + +from __future__ import annotations + +import asyncio +import json +import os +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from dotenv import load_dotenv +from mcp import ClientSession +from mcp.client.streamable_http import streamablehttp_client + +from judge.report import ( + RESULTS_DIR, + _competing_tool_hint, + resolve_default_summary_path, + resolve_suggestion_output_path, + run_dir_for_summary_path, +) + +load_dotenv() + +DEFAULT_SUMMARY_PATH = RESULTS_DIR / "latest_summary.json" +DEFAULT_OUTPUT_PATH = RESULTS_DIR / "suggested_overrides.json" + +DEFAULT_CASE_TYPES = frozenset({"ambiguous_selection", "happy_path", "missing_parameter", "multi_tool"}) + +AMBIGUOUS_PROMPT = """\ +You are improving MCP tool descriptions so an AI agent routes user requests to the correct tool. + +An ambiguous_selection eval failed: the agent chose the wrong tool. + +Eval case id: {case_id} +Case intent: {case_description} +User prompt: {eval_prompt} +Expected tool: {expected_tool} +Tool the agent chose instead: {actual_tool} +Failure: {failure_detail} +{judge_notes_block} + +Current MCP descriptions: + +[{expected_tool}] +{expected_description} + +[{actual_tool}] +{actual_description} + +{existing_override_block} + +Rewrite BOTH descriptions so only `{expected_tool}` fits the prompt and `{actual_tool}` clearly does not. +Rules: state when to use and explicit "Do NOT use when…" limits; stay faithful to Teradata MCP behavior; +2–4 sentences each; do not copy the eval prompt verbatim. + +Respond with valid JSON only: +{{ + "tools": {{ + "{expected_tool}": "", + "{actual_tool}": "" + }}, + "rationale": "" +}} +""" + +HAPPY_PATH_PROMPT = """\ +You are improving MCP tool descriptions so an AI agent selects the correct tool and parameters. + +A happy_path eval failed: the agent did not call the expected tool correctly. + +Eval case id: {case_id} +Case intent: {case_description} +User prompt: {eval_prompt} +Expected tool: {expected_tool} +Actual tool(s) called: {actual_tools_summary} +Failure: {failure_detail} +{judge_notes_block} + +Current MCP descriptions: +{tool_description_block} + +{existing_override_block} + +Revise the description(s) for: {tool_list}. +Make `{expected_tool}` clearly match prompts like the one above. If another tool absorbed the request, \ +narrow that tool with explicit limitations. +Rules: stay faithful to Teradata MCP behavior; 2–4 sentences per tool; do not copy the eval prompt verbatim. + +Respond with valid JSON only: +{{ + "tools": {{ + "": "" + }}, + "rationale": "" +}} +Include one entry per tool listed in "Revise the description(s) for". +""" + +MISSING_PARAMETER_PROMPT = """\ +You are improving MCP tool descriptions so an AI agent asks for missing information instead of \ +calling tools prematurely or guessing. + +A missing_parameter eval failed. + +Eval case id: {case_id} +Case intent: {case_description} +User prompt: {eval_prompt} +Expected behavior: ask for clarification — do NOT call a tool yet +Actual tool(s) called: {actual_tools_summary} +Failure: {failure_detail} +{judge_notes_block} +{turn_details_block} + +Current MCP descriptions: +{tool_description_block} + +{existing_override_block} + +Revise the description(s) for: {tool_list}. +Each revised description must state which required inputs must be present before calling \ +(e.g. database_name, table_name, sql) and tell the agent to ask the user when they are missing. +Rules: stay faithful to Teradata MCP behavior; 2–4 sentences per tool; do not copy the eval prompt verbatim. + +Respond with valid JSON only: +{{ + "tools": {{ + "": "" + }}, + "rationale": "" +}} +""" + +MULTI_TOOL_PROMPT = """\ +You are improving MCP tool descriptions for multi-step workflows. + +A multi_tool eval failed: the agent did not chain tools in the expected order. + +Eval case id: {case_id} +Case intent: {case_description} +User prompt: {eval_prompt} +Expected tool order: {expected_tools_summary} +Actual tool(s) called: {actual_tools_summary} +Failure: {failure_detail} +{judge_notes_block} + +Current MCP descriptions: +{tool_description_block} + +{existing_override_block} + +Revise the description(s) for: {tool_list}. +Clarify when each tool applies in a multi-step task and what must be completed before the next tool. +Rules: stay faithful to Teradata MCP behavior; 2–4 sentences per tool; do not copy the eval prompt verbatim. + +Respond with valid JSON only: +{{ + "tools": {{ + "": "" + }}, + "rationale": "" +}} +""" + + +@dataclass +class CaseSuggestionPlan: + case_id: str + case_type: str + tool_names: list[str] + prompt: str + skip_reason: str | None = None + + +def load_summary(path: Path) -> dict[str, Any]: + if not path.exists(): + raise FileNotFoundError(f"Summary not found: {path}. Run baseline evals first.") + return json.loads(path.read_text()) + + +def load_failed_cases( + summary: dict[str, Any], + *, + case_types: frozenset[str] | None = None, +) -> list[dict[str, Any]]: + """Return failed cases from an eval summary, optionally filtered by case type.""" + allowed = case_types or DEFAULT_CASE_TYPES + cases = summary.get("cases", []) + return [ + case + for case in cases + if not case.get("passed", True) and case.get("case_type") in allowed + ] + + +def load_failed_ambiguous_cases(summary: dict[str, Any]) -> list[dict[str, Any]]: + """Return failed ambiguous_selection cases (backward-compatible helper).""" + return load_failed_cases(summary, case_types=frozenset({"ambiguous_selection"})) + + +def _tool_name(blocks: list[dict[str, Any]] | None, *, fallback: str = "") -> str: + if not blocks: + return fallback + return blocks[0].get("name", fallback) + + +def _tool_names_from_blocks(blocks: list[dict[str, Any]] | None) -> list[str]: + if not blocks: + return [] + return [tool["name"] for tool in blocks if tool.get("name")] + + +def tool_pair_for_case(case: dict[str, Any]) -> tuple[str, str] | None: + """Return (expected_tool, competing_tool) for an ambiguous_selection failure.""" + expected = _tool_name(case.get("expected_tools")) + if not expected: + return None + + actual = _tool_name(case.get("actual_tools")) + if actual and actual != expected: + return expected, actual + + competing = _competing_tool_hint(case.get("description", ""), expected) + if competing and competing != expected: + return expected, competing + + if actual: + return expected, actual + + return None + + +def _format_tool_list(names: list[str]) -> str: + return ", ".join(f"`{name}`" for name in names) if names else "(none)" + + +def _format_tools_summary(blocks: list[dict[str, Any]] | None) -> str: + if not blocks: + return "none" + parts = [] + for tool in blocks: + params = json.dumps(tool.get("params", {}), sort_keys=True) + parts.append(f"{tool['name']}({params})") + return "; ".join(parts) + + +def _existing_override_block(tool_names: list[str], existing_overrides: dict[str, str]) -> str: + if not existing_overrides: + return "" + lines = ["Existing local overrides (may be empty for these tools):"] + for name in tool_names: + if name in existing_overrides: + lines.append(f"[{name}] {existing_overrides[name]}") + if len(lines) == 1: + return "" + return "\n".join(lines) + + +def _judge_notes_block(case: dict[str, Any]) -> str: + reasons = case.get("metric_reasons") or [] + if not reasons: + return "" + return "Judge notes:\n" + "\n".join(f"- {reason}" for reason in reasons) + + +def _turn_details_block(case: dict[str, Any]) -> str: + turn_details = case.get("turn_details") + if not turn_details: + return "" + return "Turn details:\n" + json.dumps(turn_details, indent=2) + + +def _tool_description_block(tool_names: list[str], live_descriptions: dict[str, str]) -> str: + blocks = [] + for name in tool_names: + blocks.append(f"[{name}]\n{live_descriptions.get(name, '(no description on MCP server)')}") + return "\n\n".join(blocks) + + +def tools_for_case(case: dict[str, Any]) -> list[str]: + """Return tool names whose descriptions should be revised for this failure.""" + case_type = case.get("case_type", "happy_path") + expected = _tool_names_from_blocks(case.get("expected_tools")) + actual = _tool_names_from_blocks(case.get("actual_tools")) + + if case_type == "ambiguous_selection": + pair = tool_pair_for_case(case) + return list(pair) if pair else [] + + if case_type == "happy_path": + names = list(dict.fromkeys(expected + [n for n in actual if n not in expected])) + return names or expected + + if case_type == "missing_parameter": + if actual: + return actual + # Clarification-only failure with no tool calls — nothing to patch in descriptions. + return [] + + if case_type == "multi_tool": + return list(dict.fromkeys(expected + [n for n in actual if n not in expected])) + + return [] + + +def build_case_plan( + case: dict[str, Any], + *, + live_descriptions: dict[str, str], + existing_overrides: dict[str, str], +) -> CaseSuggestionPlan | None: + """Build the Bedrock prompt plan for one failed case, or None if not actionable.""" + case_id = case.get("case_id", "") + case_type = case.get("case_type", "happy_path") + tool_names = tools_for_case(case) + + if case_type == "missing_parameter" and not tool_names: + return CaseSuggestionPlan( + case_id=case_id, + case_type=case_type, + tool_names=[], + prompt="", + skip_reason="No tool calls to address — failure is clarification wording, not tool routing.", + ) + + if not tool_names: + return None + + common = { + "case_id": case_id, + "case_description": case.get("description", ""), + "eval_prompt": case.get("input", ""), + "failure_detail": case.get("failure_detail") or "; ".join(case.get("metric_reasons") or []) or "unknown", + "judge_notes_block": _judge_notes_block(case), + "existing_override_block": _existing_override_block(tool_names, existing_overrides), + "tool_description_block": _tool_description_block(tool_names, live_descriptions), + "tool_list": _format_tool_list(tool_names), + } + + if case_type == "ambiguous_selection": + expected_tool, actual_tool = tool_names[0], tool_names[1] + prompt = AMBIGUOUS_PROMPT.format( + expected_tool=expected_tool, + actual_tool=actual_tool, + expected_description=live_descriptions.get(expected_tool, "(no description on MCP server)"), + actual_description=live_descriptions.get(actual_tool, "(no description on MCP server)"), + **common, + ) + elif case_type == "happy_path": + expected_tool = _tool_name(case.get("expected_tools")) + prompt = HAPPY_PATH_PROMPT.format( + expected_tool=expected_tool, + actual_tools_summary=_format_tools_summary(case.get("actual_tools")), + **common, + ) + elif case_type == "missing_parameter": + prompt = MISSING_PARAMETER_PROMPT.format( + actual_tools_summary=_format_tools_summary(case.get("actual_tools")), + turn_details_block=_turn_details_block(case), + **common, + ) + elif case_type == "multi_tool": + prompt = MULTI_TOOL_PROMPT.format( + expected_tools_summary=_format_tools_summary(case.get("expected_tools")), + actual_tools_summary=_format_tools_summary(case.get("actual_tools")), + **common, + ) + else: + return None + + return CaseSuggestionPlan( + case_id=case_id, + case_type=case_type, + tool_names=tool_names, + prompt=prompt, + ) + + +def collect_tool_names(cases: list[dict[str, Any]]) -> set[str]: + names: set[str] = set() + for case in cases: + names.update(tools_for_case(case)) + return names + + +async def fetch_live_descriptions(mcp_url: str, tool_names: set[str]) -> dict[str, str]: + """Fetch live MCP descriptions for the requested tool names.""" + if not tool_names: + return {} + + async with streamablehttp_client(mcp_url) as (read, write, _): + async with ClientSession(read, write) as session: + await session.initialize() + result = await session.list_tools() + + descriptions: dict[str, str] = {} + for tool in result.tools: + if tool.name in tool_names: + descriptions[tool.name] = tool.description or "" + return descriptions + + +def load_existing_overrides(path: Path | None) -> dict[str, str]: + if path is None or not path.exists(): + return {} + try: + data = json.loads(path.read_text()) + if isinstance(data, dict): + return {k: v for k, v in data.items() if isinstance(v, str) and not k.startswith("_")} + except Exception: + pass + return {} + + +def parse_suggestion_response(text: str) -> dict[str, Any]: + """Extract the JSON object from an LLM response.""" + start = text.index("{") + end = text.rindex("}") + 1 + parsed = json.loads(text[start:end]) + if not isinstance(parsed, dict) or "tools" not in parsed: + raise ValueError("LLM response missing 'tools' object") + return parsed + + +def suggest_for_plan( + plan: CaseSuggestionPlan, + *, + bedrock_client, + model_id: str, +) -> dict[str, Any]: + response = bedrock_client.converse( + modelId=model_id, + messages=[{"role": "user", "content": [{"text": plan.prompt}]}], + ) + text = response["output"]["message"]["content"][0]["text"].strip() + return parse_suggestion_response(text) + + +def merge_suggestions( + *, + source_summary: Path, + case_results: list[dict[str, Any]], + skipped: list[dict[str, str]] | None = None, +) -> dict[str, Any]: + """Build the output document written to suggested_overrides.json.""" + merged_tools: dict[str, str] = {} + for entry in case_results: + for name, description in entry.get("tools", {}).items(): + merged_tools[name] = description + + payload: dict[str, Any] = { + "_comment": "LLM draft — review before running suggest_overrides.py --apply", + "_generated_at": datetime.now(timezone.utc).isoformat(), + "_source_summary": str(source_summary), + "_supported_case_types": sorted(DEFAULT_CASE_TYPES), + "_instructions": ( + "Review this draft, then run: uv run python suggest_overrides.py --apply " + "and re-run evals with: uv run python run_evals.py --with-description-overrides" + ), + "suggestions": merged_tools, + "cases": case_results, + } + run_dir = run_dir_for_summary_path(source_summary) + if run_dir is not None: + payload["_source_run_id"] = run_dir.name + payload["_source_run_dir"] = str(run_dir.relative_to(RESULTS_DIR)) + if skipped: + payload["skipped"] = skipped + return payload + + +DEFAULT_OVERRIDES_COMMENT = ( + "Dev-space description overrides. Gitignored. Edit freely, then promote to MCP server." +) + + +def load_suggestion_draft(path: Path) -> dict[str, str]: + """Return tool-name → description entries from a suggested_overrides.json draft.""" + if not path.exists(): + raise FileNotFoundError(f"Suggestions file not found: {path}") + + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError(f"Invalid suggestions file (expected JSON object): {path}") + + suggestions = data.get("suggestions", {}) + if not isinstance(suggestions, dict): + raise ValueError(f"Invalid suggestions file (missing 'suggestions' object): {path}") + + return {name: text for name, text in suggestions.items() if isinstance(name, str) and isinstance(text, str)} + + +def apply_suggestions_to_overrides( + *, + suggestions_path: Path | None = None, + overrides_path: Path | None = None, + only_tools: set[str] | None = None, + dry_run: bool = False, +) -> dict[str, Any]: + """Replace description_overrides.json with reviewed suggestions only.""" + from agent.client import DEFAULT_OVERRIDES_FILE + + resolved_suggestions = suggestions_path or resolve_suggestion_output_path(resolve_default_summary_path()) + target = overrides_path or DEFAULT_OVERRIDES_FILE + suggestions = load_suggestion_draft(resolved_suggestions) + if only_tools is not None: + suggestions = {name: text for name, text in suggestions.items() if name in only_tools} + + if not suggestions: + return { + "message": "No suggestions to apply.", + "suggestions_path": str(resolved_suggestions), + "overrides_path": str(target), + "applied": [], + "removed": [], + } + + previous_tools: set[str] = set() + comment = DEFAULT_OVERRIDES_COMMENT + if target.exists(): + previous_raw = json.loads(target.read_text(encoding="utf-8")) + if not isinstance(previous_raw, dict): + raise ValueError(f"Invalid overrides file (expected JSON object): {target}") + previous_tools = { + key for key, value in previous_raw.items() if isinstance(value, str) and not key.startswith("_") + } + if isinstance(previous_raw.get("_comment"), str): + comment = previous_raw["_comment"] + + applied = sorted(suggestions) + removed = sorted(previous_tools - set(suggestions)) + raw = {"_comment": comment, **suggestions} + + if not dry_run: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(raw, indent=2) + "\n", encoding="utf-8") + + return { + "suggestions_path": str(resolved_suggestions), + "overrides_path": str(target), + "applied": applied, + "removed": removed, + "applied_count": len(suggestions), + } + + +def _parse_case_types(case_type: str | None) -> frozenset[str]: + if not case_type: + return DEFAULT_CASE_TYPES + selected = {part.strip() for part in case_type.split(",") if part.strip()} + unknown = selected - DEFAULT_CASE_TYPES + if unknown: + allowed = ", ".join(sorted(DEFAULT_CASE_TYPES)) + raise ValueError(f"Unknown case type(s): {', '.join(sorted(unknown))}. Allowed: {allowed}") + return frozenset(selected) + + +def generate_suggestions( + *, + summary_path: Path | None = None, + output_path: Path | None = None, + overrides_path: Path | None = None, + mcp_url: str | None = None, + model_id: str | None = None, + case_type: str | None = None, + dry_run: bool = False, +) -> dict[str, Any]: + """Generate draft override descriptions for failed eval cases.""" + import boto3 + + resolved_summary = summary_path or resolve_default_summary_path() + resolved_output = output_path or resolve_suggestion_output_path(resolved_summary) + + summary = load_summary(resolved_summary) + allowed_types = _parse_case_types(case_type) + failed_cases = load_failed_cases(summary, case_types=allowed_types) + if not failed_cases: + allowed = ", ".join(sorted(allowed_types)) + return { + "suggestions": {}, + "cases": [], + "message": f"No failed cases in summary for type(s): {allowed}.", + } + + tool_names = collect_tool_names(failed_cases) + resolved_url = mcp_url or os.environ.get("MCP_SERVER_URL", "http://127.0.0.1:8001/mcp") + live_descriptions = asyncio.run(fetch_live_descriptions(resolved_url, tool_names)) + + missing = sorted(name for name in tool_names if name not in live_descriptions) + if missing: + raise RuntimeError(f"Could not fetch live descriptions for: {', '.join(missing)}") + + existing_overrides = load_existing_overrides(overrides_path) + resolved_model = model_id or os.environ.get( + "BEDROCK_JUDGE_MODEL_ID", + os.environ.get("BEDROCK_MODEL_ID", "anthropic.claude-3-5-sonnet-20241022-v2:0"), + ) + region = os.environ.get("AWS_REGION", "us-east-1") + bedrock_client = boto3.client("bedrock-runtime", region_name=region) + + case_results: list[dict[str, Any]] = [] + skipped: list[dict[str, str]] = [] + + for case in failed_cases: + plan = build_case_plan(case, live_descriptions=live_descriptions, existing_overrides=existing_overrides) + if plan is None: + continue + if plan.skip_reason: + skipped.append({"case_id": plan.case_id, "case_type": plan.case_type, "reason": plan.skip_reason}) + continue + + if dry_run: + case_results.append( + { + "case_id": plan.case_id, + "case_type": plan.case_type, + "tool_names": plan.tool_names, + "eval_prompt": case.get("input"), + "prompt_preview": plan.prompt, + } + ) + continue + + parsed = suggest_for_plan(plan, bedrock_client=bedrock_client, model_id=resolved_model) + tools = parsed.get("tools", {}) + if not isinstance(tools, dict): + raise ValueError(f"Invalid tools payload for case {plan.case_id}") + + case_results.append( + { + "case_id": plan.case_id, + "case_type": plan.case_type, + "tool_names": plan.tool_names, + "eval_prompt": case.get("input"), + "tools": {k: v for k, v in tools.items() if isinstance(v, str)}, + "rationale": parsed.get("rationale", ""), + } + ) + + if not case_results and not skipped: + return { + "suggestions": {}, + "cases": [], + "message": "Failed cases found but none were actionable for description overrides.", + } + + payload = merge_suggestions(source_summary=resolved_summary, case_results=case_results, skipped=skipped or None) + if not dry_run and (case_results or skipped): + resolved_output.parent.mkdir(parents=True, exist_ok=True) + resolved_output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + legacy_output = RESULTS_DIR / "suggested_overrides.json" + if resolved_output != legacy_output: + legacy_output.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + + return payload diff --git a/tests/eval_test/preflight.py b/tests/eval_test/preflight.py new file mode 100644 index 0000000..ae94ac7 --- /dev/null +++ b/tests/eval_test/preflight.py @@ -0,0 +1,74 @@ +"""Verify eval prerequisites before running live evals against Teradata.""" + +from __future__ import annotations + +import os +import sys + +from dotenv import load_dotenv + +load_dotenv() + +EVAL_TABLES = ("evals_employees", "evals_orders") + + +def check_eval_tables() -> list[str]: + """Return human-readable errors; empty list means OK.""" + db = os.environ.get("EVALS_DATABASE", "").strip() + if not db: + return ["EVALS_DATABASE is not set — copy .env.example to .env and fill in values"] + + missing_env = [ + name + for name in ("TERADATA_HOST", "TERADATA_USER", "TERADATA_PASSWORD") + if not os.environ.get(name, "").strip() + ] + if missing_env: + return [f"Missing Teradata env var(s): {', '.join(missing_env)}"] + + try: + from setup_test_data import get_connection + except ImportError as exc: + return [f"Cannot import Teradata helpers: {exc}"] + + errors: list[str] = [] + try: + with get_connection() as con: + with con.cursor() as cur: + for table in EVAL_TABLES: + try: + cur.execute(f"SELECT COUNT(*) FROM {db}.{table}") + row_count = cur.fetchone()[0] + except Exception as exc: + errors.append(f"{db}.{table} is not accessible: {exc}") + continue + if row_count == 0: + errors.append(f"{db}.{table} exists but has no rows") + except Exception as exc: + errors.append(f"Cannot connect to Teradata ({os.environ.get('TERADATA_HOST')}): {exc}") + + if errors: + errors.append("Fix with: uv run python setup_test_data.py") + + return errors + + +def run_preflight() -> None: + errors = check_eval_tables() + if not errors: + db = os.environ.get("EVALS_DATABASE", "").strip() + print(f"Preflight OK — {db}.evals_employees and {db}.evals_orders are ready.") + return + + print("Preflight failed:", file=sys.stderr) + for error in errors: + print(f" - {error}", file=sys.stderr) + raise SystemExit(1) + + +def main() -> None: + run_preflight() + + +if __name__ == "__main__": + main() diff --git a/tests/eval_test/pyproject.toml b/tests/eval_test/pyproject.toml new file mode 100644 index 0000000..d632039 --- /dev/null +++ b/tests/eval_test/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "teradata-mcp-evals" +version = "0.1.0" +description = "Eval suite for the Teradata MCP Server community edition" +requires-python = ">=3.11" +dependencies = [ + "deepeval>=1.4.0", + "mcp>=1.0.0", + "boto3>=1.34.0", + "python-dotenv>=1.0.0", + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", + "teradatasql>=20.0.0", +] + +[project.scripts] +run-evals = "run_evals:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["agent", "judge"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.ruff] +line-length = 120 diff --git a/tests/eval_test/results/index.json b/tests/eval_test/results/index.json new file mode 100644 index 0000000..05ae1dc --- /dev/null +++ b/tests/eval_test/results/index.json @@ -0,0 +1,46 @@ +{ + "runs": [ + { + "run_id": "2026-06-24T00-06-59.222908Z__base__baseline", + "started_at": "2026-06-24T00:06:59.222908+00:00", + "module_filter": "base", + "case_type_filter": "all", + "run_label": null, + "description_mode": "mcp_server", + "passed": 0, + "failed": 32, + "total": 32, + "run_dir": "runs/2026-06-24T00-06-59.222908Z__base__baseline", + "summary_json": "runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json", + "summary_md": "runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md" + }, + { + "run_id": "2026-06-24T00-05-13.720459Z__base__baseline", + "started_at": "2026-06-24T00:05:13.720459+00:00", + "module_filter": "base", + "case_type_filter": "all", + "run_label": null, + "description_mode": "mcp_server", + "passed": 0, + "failed": 32, + "total": 32, + "run_dir": "runs/2026-06-24T00-05-13.720459Z__base__baseline", + "summary_json": "runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.json", + "summary_md": "runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.md" + }, + { + "run_id": "2026-06-23T21-14-19.042368Z__all__baseline", + "started_at": "2026-06-23T21:14:19.042368+00:00", + "module_filter": "all", + "case_type_filter": "all", + "run_label": null, + "description_mode": "mcp_server", + "passed": 103, + "failed": 3, + "total": 106, + "run_dir": "runs/2026-06-23T21-14-19.042368Z__all__baseline", + "summary_json": "runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.json", + "summary_md": "runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.md" + } + ] +} diff --git a/tests/eval_test/results/latest.json b/tests/eval_test/results/latest.json new file mode 100644 index 0000000..88f35c5 --- /dev/null +++ b/tests/eval_test/results/latest.json @@ -0,0 +1,15 @@ +{ + "run_id": "2026-06-24T00-06-59.222908Z__base__baseline", + "started_at": "2026-06-24T00:06:59.222908+00:00", + "run_dir": "runs/2026-06-24T00-06-59.222908Z__base__baseline", + "summary_md": "runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md", + "summary_json": "runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json", + "manifest": "runs/2026-06-24T00-06-59.222908Z__base__baseline/manifest.json", + "module_filter": "base", + "case_type_filter": "all", + "run_label": null, + "description_mode": "mcp_server", + "passed": 0, + "failed": 32, + "total": 32 +} diff --git a/tests/eval_test/results/latest_summary.json b/tests/eval_test/results/latest_summary.json new file mode 100644 index 0000000..90bd6d6 --- /dev/null +++ b/tests/eval_test/results/latest_summary.json @@ -0,0 +1,683 @@ +{ + "run_id": "2026-06-24T00-06-59.222908Z__base__baseline", + "started_at": "2026-06-24T00:06:59.222908+00:00", + "module_filter": "base", + "case_type_filter": "all", + "run_label": null, + "agent_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "judge_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "evals_database": "demo_user", + "description_mode": "mcp_server", + "description_overrides_file": null, + "description_override_count": 0, + "total": 32, + "passed": 0, + "failed": 32, + "cases": [ + { + "case_id": "base_readQuery_happy", + "case_type": "happy_path", + "description": "Execute a SQL query against the evals employees table", + "input": "Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_readQuery_ambiguous_vs_tablePreview", + "case_type": "ambiguous_selection", + "description": "Prompt with a WHERE clause should use base_readQuery, not base_tablePreview", + "input": "Get me all orders from demo_user.evals_orders where the amount is greater than 500", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_readQuery_missing_sql", + "case_type": "missing_parameter", + "description": "Agent must ask for clarification when no SQL or table is given", + "input": "Query the database for me", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_happy", + "case_type": "happy_path", + "description": "List tables in the evals database", + "input": "What tables are in the demo_user database?", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_missing_database", + "case_type": "missing_parameter", + "description": "User asks for tables without specifying a database", + "input": "Show me what tables are available", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableDDL_happy", + "case_type": "happy_path", + "description": "Retrieve DDL for the evals employees table", + "input": "Show me the CREATE statement for evals_employees in the demo_user database", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableDDL_ambiguous_vs_columnMetadata", + "case_type": "ambiguous_selection", + "description": "Asking for full structure including constraints should prefer base_tableDDL", + "input": "Describe the full structure of demo_user.evals_employees including any constraints and indexes", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnMetadata_happy", + "case_type": "happy_path", + "description": "Get precise Teradata type codes, character sets, and precision for columns in evals_orders", + "input": "Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnMetadata_ambiguous_vs_tableDDL", + "case_type": "ambiguous_selection", + "description": "Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL", + "input": "I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_happy", + "case_type": "happy_path", + "description": "Preview a sample of rows from the evals orders table", + "input": "Give me a quick preview of what's in demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_ambiguous_vs_readQuery", + "case_type": "ambiguous_selection", + "description": "A simple 'show me a few rows' without a WHERE clause should use base_tablePreview", + "input": "Show me a few rows from demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_databaseList_happy", + "case_type": "happy_path", + "description": "List all accessible databases", + "input": "What databases do I have access to?", + "expected_tools": [ + { + "name": "base_databaseList", + "params": {} + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_ambiguous_vs_databaseList", + "case_type": "ambiguous_selection", + "description": "Listing tables inside one database should prefer base_tableList, not base_databaseList", + "input": "What tables exist inside the demo_user database?", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_databaseList_ambiguous_vs_tableList", + "case_type": "ambiguous_selection", + "description": "Listing accessible databases should prefer base_databaseList, not base_tableList", + "input": "Which databases am I allowed to connect to on this system?", + "expected_tools": [ + { + "name": "base_databaseList", + "params": {} + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_saveDDL_ambiguous_vs_tableDDL", + "case_type": "ambiguous_selection", + "description": "Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL", + "input": "Export the DDL for demo_user.evals_orders to a file on disk", + "expected_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableAffinity_ambiguous_vs_tableUsage", + "case_type": "ambiguous_selection", + "description": "Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage", + "input": "Which tables in demo_user tend to appear in the same queries as evals_employees?", + "expected_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableUsage_ambiguous_vs_tableAffinity", + "case_type": "ambiguous_selection", + "description": "Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity", + "input": "Who has been reading tables in demo_user lately and how often?", + "expected_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_saveDDL_happy", + "case_type": "happy_path", + "description": "Save DDL for the evals employees table to a file", + "input": "Save the DDL for evals_employees in demo_user to a file", + "expected_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableAffinity_happy", + "case_type": "happy_path", + "description": "Find tables commonly queried together with evals_orders", + "input": "Which tables in demo_user are most often queried together with evals_orders?", + "expected_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableUsage_happy", + "case_type": "happy_path", + "description": "Check usage metrics for the evals database", + "input": "How frequently are the tables in demo_user being accessed and by whom?", + "expected_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_multi_tool_list_then_preview", + "case_type": "multi_tool", + "description": "List tables in evals database then preview one", + "input": "List all tables in demo_user, then show me a preview of evals_employees", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + }, + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_multi_tool_ddl_then_columns", + "case_type": "multi_tool", + "description": "Get DDL then drill into column detail for the same table", + "input": "First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + }, + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableDDL_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask for database name when only table is provided", + "input": "Show me the CREATE statement for evals_employees", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask which table to preview", + "input": "Preview some rows", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_saveDDL_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask which table to save DDL for", + "input": "Save the DDL to a file", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which table to preview, then calls base_tablePreview after user clarifies", + "input": "Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which database, then calls base_tableList after user clarifies", + "input": "Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_readQuery_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks for SQL, then calls base_readQuery after user provides the query", + "input": "Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnDescription_happy", + "case_type": "happy_path", + "description": "Show basic column info for the evals employees table", + "input": "Show me the columns in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnDescription_ambiguous_vs_columnMetadata", + "case_type": "ambiguous_selection", + "description": "A simple 'what columns does this table have' question should prefer base_columnDescription", + "input": "What are the column names and types in demo_user.evals_orders?", + "expected_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnMetadata_ambiguous_vs_columnDescription", + "case_type": "ambiguous_selection", + "description": "Asking for precise Teradata type codes and character sets should prefer base_columnMetadata", + "input": "Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_unknown_tool_missing", + "case_type": "missing_parameter", + "description": "User asks to do something no tool supports \u2014 agent should explain and ask for clarification", + "input": "Automatically tune my Teradata database indexes and rebalance the AMPs", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + } + ] +} diff --git a/tests/eval_test/results/latest_summary.md b/tests/eval_test/results/latest_summary.md new file mode 100644 index 0000000..9973a9d --- /dev/null +++ b/tests/eval_test/results/latest_summary.md @@ -0,0 +1,867 @@ +# Teradata MCP Eval Run Summary + +**Started (UTC):** 2026-06-24T00:06:59.222908+00:00 +**Module filter:** base +**Case type filter:** all +**Agent model:** anthropic.claude-3-5-sonnet-20241022-v2:0 +**Judge model:** anthropic.claude-3-5-sonnet-20241022-v2:0 +**Eval database:** demo_user +**Tool descriptions:** live MCP server (baseline) + +## Overview + +| Metric | Count | +| --- | ---: | +| Total cases | 32 | +| Passed | 0 | +| Failed | 32 | + +## Failed cases + +### base_readQuery_happy (happy_path) + +**Description:** Execute a SQL query against the evals employees table + +**Eval prompt:** + +> Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales' + +**Expected tool(s):** + +- `base_readQuery` with params: +```json +{ + "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_readQuery_ambiguous_vs_tablePreview (ambiguous_selection) + +**Description:** Prompt with a WHERE clause should use base_readQuery, not base_tablePreview + +**Eval prompt:** + +> Get me all orders from demo_user.evals_orders where the amount is greater than 500 + +**Expected tool(s):** + +- `base_readQuery` with params: +```json +{ + "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_readQuery_missing_sql (missing_parameter) + +**Description:** Agent must ask for clarification when no SQL or table is given + +**Eval prompt:** + +> Query the database for me + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_happy (happy_path) + +**Description:** List tables in the evals database + +**Eval prompt:** + +> What tables are in the demo_user database? + +**Expected tool(s):** + +- `base_tableList` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_missing_database (missing_parameter) + +**Description:** User asks for tables without specifying a database + +**Eval prompt:** + +> Show me what tables are available + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableDDL_happy (happy_path) + +**Description:** Retrieve DDL for the evals employees table + +**Eval prompt:** + +> Show me the CREATE statement for evals_employees in the demo_user database + +**Expected tool(s):** + +- `base_tableDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableDDL_ambiguous_vs_columnMetadata (ambiguous_selection) + +**Description:** Asking for full structure including constraints should prefer base_tableDDL + +**Eval prompt:** + +> Describe the full structure of demo_user.evals_employees including any constraints and indexes + +**Expected tool(s):** + +- `base_tableDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnMetadata_happy (happy_path) + +**Description:** Get precise Teradata type codes, character sets, and precision for columns in evals_orders + +**Eval prompt:** + +> Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders + +**Expected tool(s):** + +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnMetadata_ambiguous_vs_tableDDL (ambiguous_selection) + +**Description:** Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL + +**Eval prompt:** + +> I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees + +**Expected tool(s):** + +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_happy (happy_path) + +**Description:** Preview a sample of rows from the evals orders table + +**Eval prompt:** + +> Give me a quick preview of what's in demo_user.evals_orders + +**Expected tool(s):** + +- `base_tablePreview` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_ambiguous_vs_readQuery (ambiguous_selection) + +**Description:** A simple 'show me a few rows' without a WHERE clause should use base_tablePreview + +**Eval prompt:** + +> Show me a few rows from demo_user.evals_orders + +**Expected tool(s):** + +- `base_tablePreview` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_databaseList_happy (happy_path) + +**Description:** List all accessible databases + +**Eval prompt:** + +> What databases do I have access to? + +**Expected tool(s):** + +- `base_databaseList` with params: +```json +{} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_ambiguous_vs_databaseList (ambiguous_selection) + +**Description:** Listing tables inside one database should prefer base_tableList, not base_databaseList + +**Eval prompt:** + +> What tables exist inside the demo_user database? + +**Expected tool(s):** + +- `base_tableList` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_databaseList_ambiguous_vs_tableList (ambiguous_selection) + +**Description:** Listing accessible databases should prefer base_databaseList, not base_tableList + +**Eval prompt:** + +> Which databases am I allowed to connect to on this system? + +**Expected tool(s):** + +- `base_databaseList` with params: +```json +{} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_saveDDL_ambiguous_vs_tableDDL (ambiguous_selection) + +**Description:** Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL + +**Eval prompt:** + +> Export the DDL for demo_user.evals_orders to a file on disk + +**Expected tool(s):** + +- `base_saveDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableAffinity_ambiguous_vs_tableUsage (ambiguous_selection) + +**Description:** Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage + +**Eval prompt:** + +> Which tables in demo_user tend to appear in the same queries as evals_employees? + +**Expected tool(s):** + +- `base_tableAffinity` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableUsage_ambiguous_vs_tableAffinity (ambiguous_selection) + +**Description:** Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity + +**Eval prompt:** + +> Who has been reading tables in demo_user lately and how often? + +**Expected tool(s):** + +- `base_tableUsage` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_saveDDL_happy (happy_path) + +**Description:** Save DDL for the evals employees table to a file + +**Eval prompt:** + +> Save the DDL for evals_employees in demo_user to a file + +**Expected tool(s):** + +- `base_saveDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableAffinity_happy (happy_path) + +**Description:** Find tables commonly queried together with evals_orders + +**Eval prompt:** + +> Which tables in demo_user are most often queried together with evals_orders? + +**Expected tool(s):** + +- `base_tableAffinity` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableUsage_happy (happy_path) + +**Description:** Check usage metrics for the evals database + +**Eval prompt:** + +> How frequently are the tables in demo_user being accessed and by whom? + +**Expected tool(s):** + +- `base_tableUsage` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_multi_tool_list_then_preview (multi_tool) + +**Description:** List tables in evals database then preview one + +**Eval prompt:** + +> List all tables in demo_user, then show me a preview of evals_employees + +**Expected tool(s):** + +- `base_tableList` with params: +```json +{ + "database_name": "demo_user" +} +``` +- `base_tablePreview` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_multi_tool_ddl_then_columns (multi_tool) + +**Description:** Get DDL then drill into column detail for the same table + +**Eval prompt:** + +> First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata + +**Expected tool(s):** + +- `base_tableDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableDDL_missing_table (missing_parameter) + +**Description:** Agent must ask for database name when only table is provided + +**Eval prompt:** + +> Show me the CREATE statement for evals_employees + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_missing_table (missing_parameter) + +**Description:** Agent must ask which table to preview + +**Eval prompt:** + +> Preview some rows + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_saveDDL_missing_table (missing_parameter) + +**Description:** Agent must ask which table to save DDL for + +**Eval prompt:** + +> Save the DDL to a file + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_clarify_then_call (missing_parameter) + +**Description:** Agent asks which table to preview, then calls base_tablePreview after user clarifies + +**Eval prompt:** + +> Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_clarify_then_call (missing_parameter) + +**Description:** Agent asks which database, then calls base_tableList after user clarifies + +**Eval prompt:** + +> Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_readQuery_clarify_then_call (missing_parameter) + +**Description:** Agent asks for SQL, then calls base_readQuery after user provides the query + +**Eval prompt:** + +> Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales' + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnDescription_happy (happy_path) + +**Description:** Show basic column info for the evals employees table + +**Eval prompt:** + +> Show me the columns in demo_user.evals_employees + +**Expected tool(s):** + +- `base_columnDescription` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnDescription_ambiguous_vs_columnMetadata (ambiguous_selection) + +**Description:** A simple 'what columns does this table have' question should prefer base_columnDescription + +**Eval prompt:** + +> What are the column names and types in demo_user.evals_orders? + +**Expected tool(s):** + +- `base_columnDescription` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnMetadata_ambiguous_vs_columnDescription (ambiguous_selection) + +**Description:** Asking for precise Teradata type codes and character sets should prefer base_columnMetadata + +**Eval prompt:** + +> Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees + +**Expected tool(s):** + +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_unknown_tool_missing (missing_parameter) + +**Description:** User asks to do something no tool supports — agent should explain and ask for clarification + +**Eval prompt:** + +> Automatically tune my Teradata database indexes and rebalance the AMPs + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +## Passed cases + +_None._ + diff --git a/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/manifest.json b/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/manifest.json new file mode 100644 index 0000000..6bed444 --- /dev/null +++ b/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/manifest.json @@ -0,0 +1,22 @@ +{ + "run_id": "2026-06-23T21-14-19.042368Z__all__baseline", + "started_at": "2026-06-23T21:14:19.042368+00:00", + "module_filter": "all", + "case_type_filter": "all", + "run_label": null, + "description_mode": "mcp_server", + "description_overrides_file": null, + "description_override_count": 0, + "agent_model_id": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "judge_model_id": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "evals_database": "demo_user", + "total": 106, + "passed": 103, + "failed": 3, + "artifacts": { + "summary_md": "summary.md", + "summary_json": "summary.json", + "manifest": "manifest.json", + "suggested_overrides": "suggested_overrides.json" + } +} diff --git a/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.json b/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.json new file mode 100644 index 0000000..6b18f98 --- /dev/null +++ b/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.json @@ -0,0 +1,3209 @@ +{ + "run_id": "2026-06-23T21-14-19.042368Z__all__baseline", + "started_at": "2026-06-23T21:14:19.042368+00:00", + "module_filter": "all", + "case_type_filter": "all", + "run_label": null, + "agent_model_id": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "judge_model_id": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", + "evals_database": "demo_user", + "description_mode": "mcp_server", + "description_overrides_file": null, + "description_override_count": 0, + "total": 106, + "passed": 103, + "failed": 3, + "cases": [ + { + "case_id": "base_readQuery_happy", + "case_type": "happy_path", + "description": "Execute a SQL query against the evals employees table", + "input": "Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" + } + } + ], + "actual_output": "Here are the results from the query:\n\n| Name | Department | Salary |\n|------|-----------|--------|\n| Bob Smith | Sales | $72,000 |\n| Frank Lee | Sales | $69,500 |\n\nThe query returned **2 employees** in the Sales department. Bob Smith has the highest salary at $72,000, followed by Frank Lee at $69,500.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_readQuery_ambiguous_vs_tablePreview", + "case_type": "ambiguous_selection", + "description": "Prompt with a WHERE clause should use base_readQuery, not base_tablePreview", + "input": "Get me all orders from demo_user.evals_orders where the amount is greater than 500", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" + } + } + ], + "actual_output": "I found **8 orders** from demo_user.evals_orders where the amount is greater than 500:\n\n| Order ID | Customer Name | Order Date | Ship Date | Amount | Product Category | Quantity |\n|----------|--------------|------------|-----------|---------|-----------------|----------|\n| 1009 | Iota Inc | 2024-05-08 | - | $1,780.00 | Electronics | 1 |\n| 1005 | Epsilon LLC | 2024-03-01 | 2024-03-04 | $560.00 | Books | 8 |\n| 1010 | Kappa Ltd | 2024-05-25 | 2024-05-28 | $4,200.00 | Sports | 7 |\n| 1007 | Eta Group | 2024-04-02 | 2024-04-06 | $2,100.00 | Sports | 4 |\n| 1001 | Acme Corp | 2024-01-05 | 2024-01-08 | $1,250.00 | Electronics | 3 |\n| 1008 | Theta Corp | 2024-04-20 | 2024-04-23 | $930.00 | Food | 6 |\n| 1004 | Delta Co | 2024-02-17 | 2024-02-20 | $3,400.00 | Electronics | 2 |\n| 1003 | Gamma Inc | 2024-02-03 | - | $875.50 | Food | 5 |\n\nThe highest value order is from **Kappa Ltd** at $4,200.00, and the orders range from $560.00 to $4,200.00.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_readQuery_missing_sql", + "case_type": "missing_parameter", + "description": "Agent must ask for clarification when no SQL or table is given", + "input": "Query the database for me", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you query the database! However, I need a bit more information to proceed:\n\n1. **What would you like to query?** For example:\n - A specific table (e.g., \"show me data from the sales table\")\n - List available databases or tables\n - Get information about table structure/columns\n - Run a specific SQL query you have in mind\n - View sample data from a table\n - Get statistics or summaries\n\n2. **Which database and/or table** are you interested in? (if you know the names)\n\nCould you provide more details about what you'd like to see or accomplish?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tableList_happy", + "case_type": "happy_path", + "description": "List tables in the evals database", + "input": "What tables are in the demo_user database?", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + } + ], + "actual_output": "The **demo_user** database contains **2 tables**:\n\n1. **evals_orders**\n2. **evals_employees**\n\nWould you like to see more details about either of these tables, such as their columns, sample data, or structure?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tableList_missing_database", + "case_type": "missing_parameter", + "description": "User asks for tables without specifying a database", + "input": "Show me what tables are available", + "expected_tools": [], + "passed": false, + "failure_stage": "deterministic", + "failure_detail": "expected no tool calls for missing_parameter case, got ['base_tableList']", + "actual_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "" + } + } + ], + "actual_output": "I can see the system has many tables available. The results show tables across multiple databases. Here's a summary of what I found:\n\n**Main Databases with Tables:**\n- **DBC** - System database with metadata tables (InDoubtLog, StatsTbl, QryLog, DBCInfo, etc.)\n- **TD_SYSFNLIB** - System function library\n- **SYSLIB** - System library\n- **TD_SERVER_DB** - Server database\n- **SQLJ** - SQL/Java support\n- **TD_SYSGPL** - System GPL (General Purpose Library)\n- **SystemFe** - System functions\n- **SYSSPATIAL** - Spatial database functions\n- **SYSUDTLIB** - User-defined types library\n- **SYSBAR** - System backup and recovery\n- **TD_SYSXML** - XML functions\n- **TDMaps** - Map functions\n- **Crashdumps** - System crash dump storage\n\nThe list shows **hundreds of system tables and views** primarily from the DBC (Data Base Control) system database, which contains metadata about databases, tables, users, permissions, query logs, and system configuration.\n\nWould you like me to:\n1. Show tables from a specific database?\n2. Filter to show only user databases (excluding system databases)?\n3. Show details about a specific table?", + "metric_reasons": [], + "recommendation": "The prompt deliberately omits required information: \"Show me what tables are available\". The agent called `base_tableList` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response.", + "turn_details": null + }, + { + "case_id": "base_tableDDL_happy", + "case_type": "happy_path", + "description": "Retrieve DDL for the evals employees table", + "input": "Show me the CREATE statement for evals_employees in the demo_user database", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tableDDL", + "params": { + "table_name": "evals_employees", + "database_name": "demo_user" + } + } + ], + "actual_output": "Here's the CREATE statement for the `evals_employees` table in the `demo_user` database:\n\n```sql\nCREATE SET TABLE demo_user.evals_employees ,FALLBACK ,\n NO BEFORE JOURNAL,\n NO AFTER JOURNAL,\n CHECKSUM = DEFAULT,\n DEFAULT MERGEBLOCKRATIO,\n MAP = TD_MAP1\n (\n employee_id INTEGER NOT NULL,\n name VARCHAR(100) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n department VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n salary DECIMAL(10,2),\n region VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n hire_date DATE FORMAT 'YY/MM/DD' NOT NULL,\n manager_id INTEGER, \nPRIMARY KEY ( employee_id ));\n```\n\nThis table contains employee information with:\n- **employee_id** (INTEGER) - Primary key\n- **name** (VARCHAR 100) - Employee name\n- **department** (VARCHAR 50) - Department name\n- **salary** (DECIMAL 10,2) - Salary amount\n- **region** (VARCHAR 50) - Geographic region\n- **hire_date** (DATE) - Date of hire\n- **manager_id** (INTEGER) - Reference to manager\n\nThe table is configured with FALLBACK for data protection and has no journal logging enabled.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tableDDL_ambiguous_vs_columnMetadata", + "case_type": "ambiguous_selection", + "description": "Asking for full structure including constraints should prefer base_tableDDL", + "input": "Describe the full structure of demo_user.evals_employees including any constraints and indexes", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tableDDL", + "params": { + "table_name": "evals_employees", + "database_name": "demo_user" + } + } + ], + "actual_output": "## Full Structure of `demo_user.evals_employees`\n\nHere's the complete DDL for the table:\n\n```sql\nCREATE SET TABLE demo_user.evals_employees ,FALLBACK ,\n NO BEFORE JOURNAL,\n NO AFTER JOURNAL,\n CHECKSUM = DEFAULT,\n DEFAULT MERGEBLOCKRATIO,\n MAP = TD_MAP1\n (\n employee_id INTEGER NOT NULL,\n name VARCHAR(100) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n department VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n salary DECIMAL(10,2),\n region VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n hire_date DATE FORMAT 'YY/MM/DD' NOT NULL,\n manager_id INTEGER, \nPRIMARY KEY ( employee_id ))\n;\n```\n\n### **Columns:**\n\n| Column | Data Type | Constraints |\n|--------|-----------|-------------|\n| **employee_id** | INTEGER | NOT NULL, PRIMARY KEY |\n| **name** | VARCHAR(100) LATIN | NOT NULL, NOT CASESPECIFIC |\n| **department** | VARCHAR(50) LATIN | NOT NULL, NOT CASESPECIFIC |\n| **salary** | DECIMAL(10,2) | Nullable |\n| **region** | VARCHAR(50) LATIN | NOT NULL, NOT CASESPECIFIC |\n| **hire_date** | DATE | NOT NULL, Format: 'YY/MM/DD' |\n| **manager_id** | INTEGER | Nullable |\n\n### **Constraints:**\n- **PRIMARY KEY**: `employee_id` - Ensures unique employee identification\n\n### **Indexes:**\n- The PRIMARY KEY on `employee_id` implicitly creates a Unique Primary Index (UPI)\n\n### **Table Attributes:**\n- **SET TABLE**: Duplicate rows are not allowed\n- **FALLBACK**: Data is protected with fallback copies\n- **NO BEFORE JOURNAL**: No before-image journaling\n- **NO AFTER JOURNAL**: No after-image journaling\n- **CHECKSUM = DEFAULT**: Uses default checksum validation\n- **DEFAULT MERGEBLOCKRATIO**: Uses system default for block merging\n- **MAP = TD_MAP1**: Uses TD_MAP1 for data distribution\n\nThis table appears to be designed for employee management with hierarchical relationships (via `manager_id`) and includes demographic and compensation information.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_columnMetadata_happy", + "case_type": "happy_path", + "description": "Get precise Teradata type codes, character sets, and precision for columns in evals_orders", + "input": "Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_orders" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_orders" + } + } + ], + "actual_output": "Here are the exact Teradata type codes, character sets, and precision details for every column in **demo_user.evals_orders**:\n\n| Column Name | Type Code | Column Type | Character Set | Length | Decimal Precision | Nullable | Index Type |\n|-------------|-----------|-------------|---------------|---------|-------------------|----------|------------|\n| **order_id** | I | INTEGER | - | 4 bytes | - | N | USI (Unique Secondary Index) |\n| **customer_name** | CV | VARCHAR(100) LATIN | LATIN | 100 | - | N | - |\n| **order_date** | DA | DATE | - | 4 bytes | - | N | - |\n| **ship_date** | DA | DATE | - | 4 bytes | - | Y | - |\n| **amount** | D | DOUBLE PRECISION | - | 8 bytes | 10 total digits, 2 fractional | N | - |\n| **product_category** | CV | VARCHAR(50) LATIN | LATIN | 50 | - | N | - |\n| **quantity** | I | INTEGER | - | 4 bytes | - | N | - |\n\n**Key Details:**\n- **Type Codes**: I = Integer, CV = Variable Character, DA = Date, D = Double Precision (Float)\n- **Character Sets**: Both VARCHAR columns use LATIN character set (CharType = 1)\n- **Decimal Precision**: The `amount` column (type D) has 10 total digits with 2 fractional digits\n- **Indexes**: `order_id` has a Unique Secondary Index (USI)\n- **Nullability**: Only `ship_date` allows NULL values; all other columns are NOT NULL", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_columnMetadata_ambiguous_vs_tableDDL", + "case_type": "ambiguous_selection", + "description": "Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL", + "input": "I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_employees" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_employees", + "fields": "ColumnName,ColumnType,Nullable,CharType" + } + } + ], + "actual_output": "Perfect! Here's the character set and nullability information for every column in **demo_user.evals_employees**:\n\n| Column Name | Data Type | Character Set | Nullable |\n|------------|-----------|---------------|----------|\n| **employee_id** | INTEGER | N/A (numeric) | **No** (NOT NULL) |\n| **name** | VARCHAR(100) | **LATIN** | **No** (NOT NULL) |\n| **department** | VARCHAR(50) | **LATIN** | **No** (NOT NULL) |\n| **salary** | DOUBLE PRECISION | N/A (numeric) | **Yes** (NULLABLE) |\n| **region** | VARCHAR(50) | **LATIN** | **No** (NOT NULL) |\n| **hire_date** | DATE | N/A (date) | **No** (NOT NULL) |\n| **manager_id** | INTEGER | N/A (numeric) | **Yes** (NULLABLE) |\n\n**Summary:**\n- **All character columns use LATIN character set** (no UNICODE columns found)\n- **Character columns:** name, department, region (all LATIN)\n- **Nullable columns:** salary, manager_id (2 columns)\n- **NOT NULL columns:** employee_id, name, department, region, hire_date (5 columns)", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tablePreview_happy", + "case_type": "happy_path", + "description": "Preview a sample of rows from the evals orders table", + "input": "Give me a quick preview of what's in demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tablePreview", + "params": { + "table_name": "evals_orders", + "database_name": "demo_user" + } + } + ], + "actual_output": "Here's a quick preview of the **demo_user.evals_orders** table:\n\n| order_id | customer_name | order_date | ship_date | amount | product_category | quantity |\n|----------|---------------|------------|-----------|---------|------------------|----------|\n| 1009 | Iota Inc | 2024-05-08 | null | 1780.0 | Electronics | 1 |\n| 1002 | Beta Ltd | 2024-01-12 | 2024-01-15 | -200.0 | Clothing | 1 |\n| 1010 | Kappa Ltd | 2024-05-25 | 2024-05-28 | 4200.0 | Sports | 7 |\n| 1008 | Theta Corp | 2024-04-20 | 2024-04-23 | 930.0 | Food | 6 |\n| 1007 | Eta Group | 2024-04-02 | 2024-04-06 | 2100.0 | Sports | 4 |\n\n**Table Structure:**\n- **order_id** (int) - Order identifier\n- **customer_name** (str) - Customer name\n- **order_date** (date) - Date order was placed\n- **ship_date** (date) - Date order was shipped (some null values)\n- **amount** (Decimal) - Order amount (note: some negative values present)\n- **product_category** (str) - Product category (Electronics, Clothing, Sports, Food, etc.)\n- **quantity** (int) - Order quantity\n\nThe table contains order transaction data with categories like Electronics, Clothing, Sports, and Food. I notice there are some null ship dates and at least one negative amount value (-200.0).", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tablePreview_ambiguous_vs_readQuery", + "case_type": "ambiguous_selection", + "description": "A simple 'show me a few rows' without a WHERE clause should use base_tablePreview", + "input": "Show me a few rows from demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tablePreview", + "params": { + "table_name": "evals_orders", + "database_name": "demo_user" + } + } + ], + "actual_output": "Here are a few sample rows from the **demo_user.evals_orders** table:\n\n| order_id | customer_name | order_date | ship_date | amount | product_category | quantity |\n|----------|---------------|------------|------------|----------|------------------|----------|\n| 1002 | Beta Ltd | 2024-01-12 | 2024-01-15 | -200.0 | Clothing | 1 |\n| 1005 | Epsilon LLC | 2024-01-12 | 2024-03-04 | 560.0 | Books | 8 |\n| 1010 | Kappa Ltd | 2024-05-25 | 2024-05-28 | 4200.0 | Sports | 7 |\n| 1008 | Theta Corp | 2024-04-20 | 2024-04-23 | 930.0 | Food | 6 |\n| 1006 | Zeta Partners | 2024-03-15 | *null* | -150.0 | Clothing | 1 |\n\nThe table contains order information with columns for order ID, customer name, order and ship dates, amount, product category, and quantity. I notice there are some negative amounts and one missing ship_date value in this sample.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_databaseList_happy", + "case_type": "happy_path", + "description": "List all accessible databases", + "input": "What databases do I have access to?", + "expected_tools": [ + { + "name": "base_databaseList", + "params": {} + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_databaseList", + "params": { + "scope": "user" + } + } + ], + "actual_output": "You have access to **10 databases** on this Teradata system:\n\n| Database Name | Type | Description |\n|--------------|------|-------------|\n| **TD_SYSXML** | DataBase | - |\n| **gs_tables_db** | DataBase | - |\n| **mldb** | User | - |\n| **HEALTHCARE_DEMO** | DataBase | - |\n| **modelops** | DataBase | - |\n| **td_tapidb** | User | - |\n| **GLOBAL_FUNCTIONS** | DataBase | - |\n| **DELTA_DEMO** | DataBase | - |\n| **demo_user** | User | Logon ID for the end user to run demonstrations |\n| **TD_SYSAI** | DataBase | - |\n\nThe databases include:\n- **User databases**: `mldb`, `td_tapidb`, and `demo_user` (user accounts with database space)\n- **Application databases**: `HEALTHCARE_DEMO`, `DELTA_DEMO`, `modelops`, `gs_tables_db`\n- **System/function databases**: `TD_SYSXML`, `TD_SYSAI`, `GLOBAL_FUNCTIONS`\n\nWould you like to explore the tables in any of these databases?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tableList_ambiguous_vs_databaseList", + "case_type": "ambiguous_selection", + "description": "Listing tables inside one database should prefer base_tableList, not base_databaseList", + "input": "What tables exist inside the demo_user database?", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + } + ], + "actual_output": "The **demo_user** database contains **2 tables**:\n\n1. **evals_orders**\n2. **evals_employees**\n\nWould you like to see more details about either of these tables, such as their column definitions or sample data?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_databaseList_ambiguous_vs_tableList", + "case_type": "ambiguous_selection", + "description": "Listing accessible databases should prefer base_databaseList, not base_tableList", + "input": "Which databases am I allowed to connect to on this system?", + "expected_tools": [ + { + "name": "base_databaseList", + "params": {} + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_databaseList", + "params": { + "scope": "user" + } + } + ], + "actual_output": "Based on the query results, you have access to **10 databases** on this Teradata system:\n\n## User Databases (User type):\n1. **mldb** - Machine Learning Database\n2. **td_tapidb** - Teradata API Database \n3. **demo_user** - Logon ID for end users to run demonstrations\n\n## Standard Databases (Database type):\n4. **TD_SYSXML** - Teradata System XML database\n5. **gs_tables_db** - Golden Standard tables database\n6. **HEALTHCARE_DEMO** - Healthcare demonstration database\n7. **modelops** - Model Operations database\n8. **GLOBAL_FUNCTIONS** - Global functions database\n9. **DELTA_DEMO** - Delta demonstration database\n10. **TD_SYSAI** - Teradata System AI database\n\nThe databases include a mix of system databases (TD_SYSXML, TD_SYSAI, GLOBAL_FUNCTIONS), machine learning and analytics databases (mldb, modelops), and demonstration databases (HEALTHCARE_DEMO, DELTA_DEMO, demo_user). You can connect to any of these databases based on your access permissions.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_saveDDL_ambiguous_vs_tableDDL", + "case_type": "ambiguous_selection", + "description": "Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL", + "input": "Export the DDL for demo_user.evals_orders to a file on disk", + "expected_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders", + "object_type": "TABLE" + } + } + ], + "actual_output": "Perfect! The DDL for `demo_user.evals_orders` has been successfully exported to disk.\n\n**File Details:**\n- **Location**: `/Users/Daniel.Tehan/Code/MCP/teradata-mcp-server/ddls_extracted/evals_orders_DDL.sql`\n- **File size**: 721 bytes\n- **DDL size**: 563 characters\n- **Object type**: TABLE\n\nThe DDL file has been saved and is ready to use.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tableAffinity_ambiguous_vs_tableUsage", + "case_type": "ambiguous_selection", + "description": "Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage", + "input": "Which tables in demo_user tend to appear in the same queries as evals_employees?", + "expected_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "actual_output": "Based on the analysis of query logs, **no tables have been found that appear in the same queries as `evals_employees`** in the `demo_user` database.\n\nThis could mean:\n- The `evals_employees` table hasn't been queried recently alongside other tables\n- Query logging may not have captured JOIN operations with this table\n- The table might be queried in isolation without JOINs to other tables\n- There may be insufficient query history in the system logs (DBC.DBQLObjTbl)\n\nIf you'd like, I can:\n1. Check if the `evals_employees` table exists and preview its structure\n2. Look at overall table usage in the `demo_user` database\n3. Examine what queries have been run against `evals_employees` specifically\n\nWould any of these help?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tableUsage_ambiguous_vs_tableAffinity", + "case_type": "ambiguous_selection", + "description": "Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity", + "input": "Who has been reading tables in demo_user lately and how often?", + "expected_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "demo_user" + } + } + ], + "actual_output": "Based on the table usage data for the **demo_user** database, here's what I found:\n\n## Recent Table Access Activity\n\nTwo tables in demo_user have been actively queried recently:\n\n### 1. **EVALS_ORDERS** (Most Popular)\n- **Query Count**: 96 queries\n- **Usage Share**: 63% of all queries\n- **Usage Frequency**: High\n- **Activity Window**: First queried 4 days ago, last queried today (0 days ago)\n\n### 2. **EVALS_EMPLOYEES**\n- **Query Count**: 55 queries\n- **Usage Share**: 36% of all queries\n- **Usage Frequency**: High\n- **Activity Window**: First queried 4 days ago, last queried today (0 days ago)\n\n## Summary\nBoth tables show **high usage frequency** with consistent access over the past 4 days, including activity today. The EVALS_ORDERS table is being accessed almost twice as frequently as EVALS_EMPLOYEES.\n\nWould you like to see more details, such as:\n- Which specific users have been running these queries?\n- What types of SQL statements are being executed?\n- Access patterns by time of day?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_saveDDL_happy", + "case_type": "happy_path", + "description": "Save DDL for the evals employees table to a file", + "input": "Save the DDL for evals_employees in demo_user to a file", + "expected_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees", + "object_type": "TABLE" + } + } + ], + "actual_output": "Perfect! The DDL for the `evals_employees` table has been successfully saved to a file.\n\n**File Details:**\n- **Location**: `/Users/Daniel.Tehan/Code/MCP/teradata-mcp-server/ddls_extracted/evals_employees_DDL.sql`\n- **File size**: 736 bytes\n- **DDL content size**: 572 characters\n\nThe DDL file contains the complete CREATE TABLE statement for the `evals_employees` table from the `demo_user` database.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tableAffinity_happy", + "case_type": "happy_path", + "description": "Find tables commonly queried together with evals_orders", + "input": "Which tables in demo_user are most often queried together with evals_orders?", + "expected_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "actual_output": "The query found no tables that are commonly queried together with `evals_orders` in the `demo_user` database. This could mean one of the following:\n\n1. **The table `evals_orders` hasn't been queried recently** - or at least not in queries that also referenced other tables\n2. **Query logging may not capture this table's usage** - the DBQLObjTbl system table only contains data if query logging is enabled and has captured queries involving this table\n3. **The table is queried in isolation** - queries may be selecting from `evals_orders` alone without JOINing to other tables\n\nTo investigate further, I can:\n- Check if the table exists and get its structure\n- Look at overall table usage patterns in the demo_user database\n- Check recent SQL queries that have accessed this table\n\nWould you like me to investigate any of these options?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tableUsage_happy", + "case_type": "happy_path", + "description": "Check usage metrics for the evals database", + "input": "How frequently are the tables in demo_user being accessed and by whom?", + "expected_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "demo_user" + } + } + ], + "actual_output": "Based on the access patterns for tables in the **demo_user** database, here's what I found:\n\n## Table Access Frequency Summary\n\n| Table Name | Query Count | % of Total | Usage Frequency | First Query | Last Query |\n|------------|------------|------------|-----------------|-------------|------------|\n| **EVALS_ORDERS** | 96 | 63% | High | 4 days ago | Today |\n| **EVALS_EMPLOYEES** | 55 | 36% | High | 4 days ago | Today |\n\n### Key Insights:\n\n1. **EVALS_ORDERS** is the most heavily accessed table, accounting for **63% of all queries** against the demo_user database with 96 total queries.\n\n2. **EVALS_EMPLOYEES** is also frequently accessed, representing **36% of queries** with 55 total queries.\n\n3. Both tables show **High usage frequency** and have been actively queried within the last 4 days, with the most recent access being today.\n\n4. These two tables represent **100% of the tracked query activity** in the demo_user database during the monitoring period.\n\nThe data shows consistent, ongoing usage of both tables, suggesting they are core tables for active business operations or analysis. Would you like me to drill down further to see which specific users are accessing these tables, or analyze the types of queries being run against them?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_multi_tool_list_then_preview", + "case_type": "multi_tool", + "description": "List tables in evals database then preview one", + "input": "List all tables in demo_user, then show me a preview of evals_employees", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + }, + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + }, + { + "name": "base_tablePreview", + "params": { + "table_name": "evals_employees", + "database_name": "demo_user" + } + } + ], + "actual_output": "## Tables in demo_user Database\n\nThe **demo_user** database contains **2 tables**:\n\n1. **evals_orders**\n2. **evals_employees**\n\n---\n\n## Preview of evals_employees Table\n\nHere's a sample of 5 rows from the **evals_employees** table:\n\n| employee_id | name | department | salary | region | hire_date | manager_id |\n|-------------|---------------|-------------|-----------|--------|------------|------------|\n| 10 | Jack Wilson | Engineering | 88,000.00 | South | 2016-08-22 | 5 |\n| 4 | David Brown | Finance | 85,000.00 | West | 2021-01-10 | 1 |\n| 5 | Eva Martinez | Engineering | 102,000.00| North | 2017-06-05 | 1 |\n| 3 | Carol White | HR | 68,000.00 | East | 2018-11-20 | 1 |\n| 7 | Grace Kim | Marketing | 77,000.00 | East | 2020-09-30 | 1 |\n\nThe table contains employee information with the following columns:\n- **employee_id**: Employee identifier\n- **name**: Employee name\n- **department**: Department (Engineering, Finance, HR, Marketing)\n- **salary**: Employee salary\n- **region**: Geographic region (North, South, East, West)\n- **hire_date**: Date when the employee was hired\n- **manager_id**: ID of the employee's manager", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_multi_tool_ddl_then_columns", + "case_type": "multi_tool", + "description": "Get DDL then drill into column detail for the same table", + "input": "First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + }, + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_orders" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tableDDL", + "params": { + "table_name": "evals_orders", + "database_name": "demo_user" + } + }, + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_orders" + } + } + ], + "actual_output": "## DDL for demo_user.evals_orders\n\n```sql\nCREATE SET TABLE demo_user.evals_orders ,FALLBACK ,\n NO BEFORE JOURNAL,\n NO AFTER JOURNAL,\n CHECKSUM = DEFAULT,\n DEFAULT MERGEBLOCKRATIO,\n MAP = TD_MAP1\n (\n order_id INTEGER NOT NULL,\n customer_name VARCHAR(100) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n order_date DATE FORMAT 'YY/MM/DD' NOT NULL,\n ship_date DATE FORMAT 'YY/MM/DD',\n amount DECIMAL(10,2) NOT NULL,\n product_category VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n quantity INTEGER NOT NULL, \nPRIMARY KEY ( order_id ))\n;\n```\n\n## Detailed Column Metadata\n\n| Column Name | Type | Length | Nullable | Character Set | Decimal (Total, Fractional) | Format | Index Type | Case Specificity | Default Value |\n|-------------|------|--------|----------|---------------|----------------------------|--------|------------|------------------|---------------|\n| **order_id** | INTEGER | 4 | No | - | - | -(10)9 | **USI** | NOT CASESPECIFIC | null |\n| **customer_name** | VARCHAR(100) LATIN | 100 | No | LATIN | - | X(100) | - | NOT CASESPECIFIC | null |\n| **order_date** | DATE | 4 | No | - | - | YY/MM/DD | - | NOT CASESPECIFIC | null |\n| **ship_date** | DATE | 4 | **Yes** | - | - | YY/MM/DD | - | NOT CASESPECIFIC | null |\n| **amount** | DOUBLE PRECISION | 8 | No | - | (10, 2) | ---------.99 | - | NOT CASESPECIFIC | null |\n| **product_category** | VARCHAR(50) LATIN | 50 | No | LATIN | - | X(50) | - | NOT CASESPECIFIC | null |\n| **quantity** | INTEGER | 4 | No | - | - | -(10)9 | - | NOT CASESPECIFIC | null |\n\n### Key Findings:\n- **Primary Key**: `order_id` with a Unique Secondary Index (USI)\n- **Total Columns**: 7\n- **Nullable Columns**: Only `ship_date` allows NULL values\n- **Character Columns**: `customer_name` and `product_category` use LATIN character set\n- **Numeric Columns**: `order_id`, `quantity` (INTEGER), and `amount` (DECIMAL with 10 total digits, 2 fractional)\n- **Date Columns**: `order_date` and `ship_date` (both formatted as YY/MM/DD)", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tableDDL_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask for database name when only table is provided", + "input": "Show me the CREATE statement for evals_employees", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I need both the table name and database name to retrieve the CREATE statement. You've provided the table name \"evals_employees\", but which database does this table belong to?\n\nCould you please provide the database name?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tablePreview_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask which table to preview", + "input": "Preview some rows", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you preview some rows from a table! However, I need to know which table you'd like to see.\n\nCould you please provide:\n1. **Table name** - The name of the table you want to preview\n2. **Database name** (optional) - The database where the table is located\n\nFor example: \"Preview rows from customer_data in the sales database\" or just \"Preview the orders table\"", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_saveDDL_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask which table to save DDL for", + "input": "Save the DDL to a file", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you save DDL to a file! However, I need some information first:\n\n1. **Database name** - Which database contains the object?\n2. **Object name** - Which table, view, or stored procedure do you want to extract DDL for?\n3. **Object type** - Is it a TABLE, VIEW, or PROCEDURE? (default is PROCEDURE)\n4. **Output directory** (optional) - Where should the file be saved? (default is './ddls_extracted')\n\nPlease provide at least the database name and object name so I can save the DDL for you.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_tablePreview_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which table to preview, then calls base_tablePreview after user clarifies", + "input": "Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_tablePreview", + "params": { + "table_name": "evals_employees", + "database_name": "demo_user" + } + } + ], + "actual_output": "Here's a preview of the **demo_user.evals_employees** table:\n\n| employee_id | name | department | salary | region | hire_date | manager_id |\n|-------------|----------------|--------------|----------|--------|------------|------------|\n| 9 | Isla Thompson | HR | *null* | North | 2023-02-14 | 3 |\n| 7 | Grace Kim | Marketing | 77000.0 | East | 2020-09-30 | 1 |\n| 4 | David Brown | Finance | 85000.0 | West | 2021-01-10 | 1 |\n| 6 | Frank Lee | Sales | 69500.0 | South | 2022-04-18 | 2 |\n| 10 | Jack Wilson | Engineering | 88000.0 | South | 2016-08-22 | 5 |\n\n**Table Overview:**\n- **7 columns**: employee_id, name, department, salary, region, hire_date, manager_id\n- Contains employee information including their department, salary, region, and hire dates\n- Note: Isla Thompson (employee_id 9) has a null salary value\n- The table appears to track employees across different departments (HR, Marketing, Finance, Sales, Engineering) and regions\n\nWould you like to see more rows or perform any analysis on this data?", + "metric_reasons": [], + "recommendation": null, + "turn_details": [ + { + "turn": 1, + "input": "Preview some rows for me", + "mode": "clarification", + "passed": true, + "actual_tools": [] + }, + { + "turn": 2, + "input": "Preview rows from demo_user.evals_employees", + "mode": "tool", + "passed": true, + "actual_tools": [ + { + "name": "base_tablePreview", + "params": { + "table_name": "evals_employees", + "database_name": "demo_user" + } + } + ] + } + ] + }, + { + "case_id": "base_tableList_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which database, then calls base_tableList after user clarifies", + "input": "Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database", + "expected_tools": [], + "passed": false, + "failure_stage": "deterministic", + "failure_detail": "turn 1: expected no tool calls for missing_parameter case, got ['base_tableList']", + "actual_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "" + } + } + ], + "actual_output": "I can see there are tables available in the system. The results show many tables from the **DBC** database (which is the Teradata Data Dictionary/system catalog). \n\nWould you like me to:\n1. **Filter by a specific database** - If you're looking for user/application tables rather than system tables, please tell me which database to show\n2. **Show only user databases** - I can list the available databases first so you can pick one\n3. **Continue showing all tables** - though the full list is quite long\n\nMost users are typically interested in their own application databases rather than the DBC system tables. Which would you prefer?", + "metric_reasons": [], + "recommendation": "The prompt deliberately omits required information: \"(multi-turn \u2014 see turn details)\". The agent called `base_tableList` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response.", + "turn_details": [ + { + "turn": 1, + "input": "Show me what tables are available", + "mode": "clarification", + "passed": false, + "failure_stage": "deterministic", + "failure_detail": "expected no tool calls for missing_parameter case, got ['base_tableList']", + "actual_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "" + } + } + ] + } + ] + }, + { + "case_id": "base_readQuery_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks for SQL, then calls base_readQuery after user provides the query", + "input": "Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'" + } + } + ], + "actual_output": "Great! Here are the results from your query:\n\n**Employees in the Sales Department:**\n\n| Name | Department |\n|------|------------|\n| Bob Smith | Sales |\n| Frank Lee | Sales |\n\n**Summary:** Found 2 employees in the Sales department.", + "metric_reasons": [], + "recommendation": null, + "turn_details": [ + { + "turn": 1, + "input": "Run a query for me", + "mode": "clarification", + "passed": true, + "actual_tools": [] + }, + { + "turn": 2, + "input": "Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'", + "mode": "tool", + "passed": true, + "actual_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'" + } + } + ] + } + ] + }, + { + "case_id": "base_columnDescription_happy", + "case_type": "happy_path", + "description": "Show basic column info for the evals employees table", + "input": "Show me the columns in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "actual_output": "The table **demo_user.evals_employees** contains the following **7 columns**:\n\n| Column Name | Data Type |\n|-------------|-----------|\n| **employee_id** | INTEGER |\n| **name** | VARCHAR |\n| **department** | VARCHAR |\n| **region** | VARCHAR |\n| **hire_date** | DATE |\n| **manager_id** | INTEGER |\n| **salary** | DECIMAL |\n\nThis appears to be an employee table with basic employee information including identification, organizational structure (department, region, manager), and compensation details.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_columnDescription_ambiguous_vs_columnMetadata", + "case_type": "ambiguous_selection", + "description": "A simple 'what columns does this table have' question should prefer base_columnDescription", + "input": "What are the column names and types in demo_user.evals_orders?", + "expected_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "actual_output": "The table `demo_user.evals_orders` contains the following columns:\n\n| Column Name | Data Type |\n|------------|-----------|\n| order_id | INTEGER |\n| order_date | DATE |\n| customer_name | VARCHAR |\n| product_category | VARCHAR |\n| quantity | INTEGER |\n| amount | DECIMAL |\n| ship_date | DATE |\n\nThe table has 7 columns in total, with a mix of date, numeric, and text fields suitable for order transaction data.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_columnMetadata_ambiguous_vs_columnDescription", + "case_type": "ambiguous_selection", + "description": "Asking for precise Teradata type codes and character sets should prefer base_columnMetadata", + "input": "Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_employees" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_employees" + } + } + ], + "actual_output": "Perfect! Here are the exact Teradata type codes, character sets, and precision details for every column in **demo_user.evals_employees**:\n\n## Column Metadata Summary\n\n| Column Name | Teradata Type Code | SQL Type String | Length | Character Set | Precision/Scale | Nullable | Index Type | Case Specificity |\n|-------------|-------------------|-----------------|---------|---------------|-----------------|----------|------------|------------------|\n| **employee_id** | **I** | INTEGER | 4 bytes | - | - | **NO** | **USI** (Unique Secondary Index) | NOT CASESPECIFIC |\n| **name** | **CV** | VARCHAR(100) LATIN | 100 chars | **LATIN** | - | **NO** | - | NOT CASESPECIFIC |\n| **department** | **CV** | VARCHAR(50) LATIN | 50 chars | **LATIN** | - | **NO** | - | NOT CASESPECIFIC |\n| **salary** | **D** | DOUBLE PRECISION | 8 bytes | - | Total: 10 digits, Fractional: 2 digits | **YES** | - | NOT CASESPECIFIC |\n| **region** | **CV** | VARCHAR(50) LATIN | 50 chars | **LATIN** | - | **NO** | - | NOT CASESPECIFIC |\n| **hire_date** | **DA** | DATE | 4 bytes | - | - | **NO** | - | NOT CASESPECIFIC |\n| **manager_id** | **I** | INTEGER | 4 bytes | - | - | **YES** | - | NOT CASESPECIFIC |\n\n### Key Details:\n\n- **Teradata Type Codes:**\n - `I` = INTEGER (4-byte signed integer)\n - `CV` = CHARACTER VARYING (VARCHAR)\n - `D` = DOUBLE PRECISION (floating-point)\n - `DA` = DATE\n\n- **Character Sets:** All string columns use **LATIN** character set (CharType=1)\n\n- **Indexes:** `employee_id` has a **Unique Secondary Index (USI)**\n\n- **Nullable Columns:** Only `salary` and `manager_id` allow NULL values\n\n- **Formats:**\n - Integers: `-(10)9` (10-digit signed format)\n - Strings: `X(n)` where n is the length\n - Salary: `---------.99` (decimal with 2 fractional digits)\n - Date: `YY/MM/DD`", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "base_unknown_tool_missing", + "case_type": "missing_parameter", + "description": "User asks to do something no tool supports \u2014 agent should explain and ask for clarification", + "input": "Automatically tune my Teradata database indexes and rebalance the AMPs", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I appreciate your interest in optimizing your Teradata database, but I need to clarify what's possible with the available tools and Teradata's architecture.\n\n## Current Tool Capabilities\n\nThe tools I have access to are **read-only analysis and monitoring tools**. They can:\n\n- \u2705 Analyze table usage patterns and query affinity\n- \u2705 Identify space usage and performance bottlenecks\n- \u2705 Review query execution history and resource consumption\n- \u2705 Examine table structures, indexes, and DDL\n- \u2705 Detect data quality issues\n- \u2705 Analyze SQL query patterns for optimization opportunities\n\n## What I Cannot Do Automatically\n\nI **cannot** execute DDL/DML operations such as:\n\n- \u274c Creating, dropping, or modifying indexes\n- \u274c Rebalancing AMPs or data distribution\n- \u274c Altering table structures\n- \u274c Changing primary indexes or partitioning\n\n## What I Can Help You With\n\nInstead, I can help you make **informed decisions** about index optimization:\n\n1. **Analyze Query Patterns**: Identify which tables are most frequently accessed and how they're queried\n2. **Review Current Indexes**: Examine existing table structures and index definitions\n3. **Identify Optimization Opportunities**: Find tables with performance issues, skew problems, or missing indexes\n4. **Generate Recommendations**: Suggest specific DDL statements you can review and execute\n\n## AMP Rebalancing Note\n\nTeradata **automatically handles AMP rebalancing** as part of its architecture. Data distribution is determined by the Primary Index (PI) hash. If you're experiencing skew issues, the solution is typically to:\n- Choose a better Primary Index column\n- Use a composite Primary Index\n- Consider partitioning strategies\n\n---\n\n**Would you like me to:**\n1. Analyze your current table usage patterns to identify optimization candidates?\n2. Review specific tables for potential index improvements?\n3. Examine query performance to find tables that would benefit from secondary indexes?\n\nPlease let me know which database you'd like me to analyze, and I'll provide actionable recommendations with the specific DDL you would need to execute.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "chat_completeChat_missing_message", + "case_type": "missing_parameter", + "description": "User invokes chat without providing a message", + "input": "Can you use the chat completion tool for me?", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I don't have a chat completion tool available in my current set of tools. The tools I have access to are primarily focused on:\n\n1. **Database operations** - querying Teradata databases, listing tables, viewing table structures\n2. **Data quality analysis** - checking for missing values, duplicates, statistics\n3. **Database administration** - monitoring space usage, user activity, sessions\n4. **Security** - checking user permissions and roles\n5. **SQL optimization** - clustering and analyzing SQL queries\n6. **Graph analysis** - analyzing database dependencies and lineage\n7. **RAG (Retrieval-Augmented Generation)** - answering questions based on document context\n8. **Visualization** - creating charts from data\n\nIf you're looking to have a conversation or need help with text generation, I can assist directly without needing a separate chat completion tool. \n\nHow can I help you? Are you looking to:\n- Work with a Teradata database?\n- Analyze data quality?\n- Visualize some data?\n- Something else?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "chat_unknown_tool_missing", + "case_type": "missing_parameter", + "description": "User asks for a non\u2011existent chat tool", + "input": "Talk to the assistant about budgeting", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you with budgeting! However, I should clarify my role and capabilities.\n\nI'm a specialized assistant designed to work with **Teradata databases**. My tools are focused on:\n\n- **Database Administration**: Analyzing disk space, table usage, system resources\n- **Data Quality**: Checking for missing values, statistics, data profiling\n- **Security**: Managing user permissions, roles, and access rights\n- **Query Analysis**: Examining SQL execution patterns, performance optimization\n- **Graph Analytics**: Dependency analysis and lineage tracing\n- **RAG (Retrieval Augmented Generation)**: Document-based question answering\n\nIf you're looking to discuss **personal or business budgeting** (managing finances, expenses, savings, etc.), I wouldn't be the right assistant for that topic, as I don't have tools designed for financial planning.\n\nHowever, if you're interested in budgeting related to:\n- **Database resource budgeting** (storage space allocation, capacity planning)\n- **Query cost analysis** (CPU/IO resource consumption)\n- **Workload management** (understanding resource usage patterns)\n\nThen I can definitely help! Could you let me know which type of budgeting you're interested in discussing?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_tableSpace_happy", + "case_type": "happy_path", + "description": "List largest tables by storage in the evals database", + "input": "Show me the 10 largest tables in demo_user ranked by storage consumption.", + "expected_tools": [ + { + "name": "dba_tableSpace", + "params": { + "database_name": "demo_user", + "top_n": 10 + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_tableSpace", + "params": { + "database_name": "demo_user", + "top_n": 10 + } + } + ], + "actual_output": "Here are the **10 largest tables in demo_user** ranked by storage consumption:\n\n| Rank | Database Name | Table Name | Current Storage (bytes) | Peak Storage (bytes) | Skew % |\n|------|---------------|------------|------------------------|---------------------|--------|\n| 1 | demo_user | **All** (Total) | 1,146,880 | 1,146,880 | 16.67% |\n| 2 | demo_user | **get_data** | 294,912 | 294,912 | 35.71% |\n| 3 | demo_user | **remove_data** | 262,144 | 262,144 | 33.33% |\n| 4 | demo_user | **evals_orders** | 196,608 | 196,608 | 0.00% |\n| 5 | demo_user | **space_report** | 196,608 | 196,608 | 25.00% |\n| 6 | demo_user | **evals_employees** | 196,608 | 196,608 | 0.00% |\n\n**Key Insights:**\n- The database contains only **5 tables** (the \"All\" row is an aggregate total)\n- **get_data** is the largest table at ~288 KB, consuming about 25.7% of total space\n- **remove_data** is second at ~256 KB (22.9% of total space)\n- The remaining three tables are roughly equal in size at ~192 KB each\n- **get_data** and **remove_data** show higher data skew (35.71% and 33.33%), suggesting uneven distribution across AMPs\n- **evals_orders** and **evals_employees** show perfect distribution with 0% skew\n\nTotal database storage: **~1.1 MB**", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_tableSqlList_happy", + "case_type": "happy_path", + "description": "List SQL executed against evals_orders over the past 14 days", + "input": "Show me all queries run against evals_orders over the last 14 days.", + "expected_tools": [ + { + "name": "dba_tableSqlList", + "params": { + "table_name": "evals_orders", + "no_days": 14 + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_tableSqlList", + "params": { + "table_name": "evals_orders", + "no_days": 14 + } + } + ], + "actual_output": "Here are all the queries run against the **evals_orders** table over the last 14 days:\n\n## Summary\nFound **16 queries** executed by **DEMO_USER** on **June 23, 2026**\n\n## Query Types\n\n### 1. **Data Retrieval Queries**\n- `SELECT * FROM demo_user.evals_orders WHERE amount > 500` (2 occurrences)\n- `SELECT TOP 5 * FROM demo_user.evals_orders` (4 occurrences)\n- `SELECT COUNT(*) FROM demo_user.evals_orders` (2 occurrences)\n\n### 2. **Metadata Queries**\n- `SHOW TABLE demo_user.evals_orders` (5 occurrences)\n\n### 3. **Table Affinity Analysis**\n- Complex query analyzing which tables are queried together with evals_orders (3 occurrences)\n - This query joins `DBC.DBQLObjTbl` with `DBC.DBQLogTbl` to find co-occurring tables in queries\n\n## Key Observations\n- All queries were executed by the **DEMO_USER** account\n- Most activity occurred on **June 23, 2026** around **20:38 - 21:17**\n- The most common operations were:\n - Previewing the table structure (`SHOW TABLE`)\n - Sampling data (`TOP 5`)\n - Filtering records by amount (`WHERE amount > 500`)\n - Analyzing table relationships (affinity analysis)\n\nAll queries appear to be exploratory in nature, focused on understanding the table structure and its relationships with other tables in the database.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_userSqlList_happy", + "case_type": "happy_path", + "description": "Show queries executed by a specific user over the past 14 days", + "input": "Show me what queries John_Smith has been executing over the past 14 days.", + "expected_tools": [ + { + "name": "dba_userSqlList", + "params": { + "user_name": "John_Smith", + "no_days": 14 + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_userSqlList", + "params": { + "user_name": "John_Smith", + "no_days": 14 + } + } + ], + "actual_output": "Based on the query results, **no SQL queries were found** for user `John_Smith` in the past 14 days.\n\nThis could mean:\n- The user hasn't executed any queries during this period\n- The username might be slightly different (case-sensitive or with different formatting)\n- The query logging may not have captured activity for this user\n- The DBQL (Database Query Logging) data may have been purged or not enabled for this user\n\nWould you like me to:\n1. Check if there are any sessions currently active for John_Smith?\n2. Try searching with a different username variation?\n3. Look at a longer time period?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_databaseSpace_happy", + "case_type": "happy_path", + "description": "Storage allocated to the evals database", + "input": "How much disk space is allocated to the demo_user database?", + "expected_tools": [ + { + "name": "dba_databaseSpace", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_databaseSpace", + "params": { + "database_name": "demo_user" + } + } + ], + "actual_output": "The **demo_user** database has the following disk space allocation:\n\n- **Space Allocated**: 78.0 GB\n- **Space Used**: 0.0 GB\n- **Free Space**: 78.0 GB\n- **Percent Used**: 0.0%\n\nThe database has 78 GB of space allocated to it, and currently none of that space is being used - it's completely empty.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_tableUsageImpact_happy", + "case_type": "happy_path", + "description": "Which users are consuming the most resources in the evals database", + "input": "Which users are consuming the most resources in demo_user?", + "expected_tools": [ + { + "name": "dba_tableUsageImpact", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_tableUsageImpact", + "params": { + "database_name": "demo_user" + } + }, + { + "name": "dba_resusageSummary", + "params": { + "no_days": 30 + } + } + ], + "actual_output": "Perfect! Based on the system-wide resource usage data, I can see that **DEMO_USER** is the primary user consuming resources. Here's a summary of their resource consumption over the last 30 days:\n\n## Resource Consumption by DEMO_USER\n\n### Summary Statistics:\n- **Primary User**: DEMO_USER (most active)\n- **Secondary User**: DBC (minimal system activity)\n\n### DEMO_USER Activity Breakdown:\n\n**Top Resource-Consuming Periods:**\n\n1. **June 19, 1:00 AM (Friday)** - Highest CPU Usage\n - 147 queries\n - **24.20 CPU seconds**\n - 3.71 GB I/O\n - Workload: Query/Answers via PYTHON3\n\n2. **June 15, 6:00 PM (Monday)**\n - 366 queries\n - **12.32 CPU seconds**\n - 3.23 GB I/O\n - Workload: Query/Answers via PYTHON3\n\n3. **June 23, 7:00 PM (Tuesday)**\n - 64 queries\n - **5.73 CPU seconds**\n - 1.97 GB I/O\n - Workload: Query/Answers via PYTHON3\n\n4. **June 8, 5:00 PM (Monday)** - ETL Activity\n - 1,957 requests\n - **3.17 CPU seconds**\n - 0.64 GB I/O\n - Workload: ETL/ELT via BTEQ\n\n### Key Insights:\n- **Application Tools**: Primarily PYTHON3, some BTEQ for ETL\n- **Workload Types**: Mostly \"Answers\" complexity queries and some ETL/System operations\n- **Pattern**: Activity spans multiple days with peak usage during evening hours (5-7 PM) and some overnight processing (1 AM)\n- **DEMO_USER** is essentially the only active user in the system (aside from minimal DBC system activity)\n\nWould you like me to drill down into specific time periods or analyze the types of queries being run?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_resusageSummary_happy", + "case_type": "happy_path", + "description": "System-wide CPU/memory/IO by application over the past month", + "input": "Show me how much CPU, memory, and I/O was consumed over the past month, grouped by application.", + "expected_tools": [ + { + "name": "dba_resusageSummary", + "params": { + "no_days": 30 + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_resusageSummary", + "params": { + "no_days": 30 + } + } + ], + "actual_output": "Based on the resource usage data from the past 30 days, here's a summary of CPU, memory, and I/O consumption grouped by application:\n\n## **Resource Consumption by Application (Past 30 Days)**\n\n### **PYTHON3 (Primary Application)**\n- **Request Count**: 1,495+ requests\n- **Total CPU Time**: 46+ seconds\n- **Total I/O Operations**: 234,546+ logical I/O\n- **Total I/O Volume**: ~15+ GB logical I/O\n- **Physical I/O**: ~132 operations, ~8 MB physical I/O\n- **Workload Types**: \n - QUERY (Answers) - majority\n - ETL/ELT (System/Procedural)\n - OTHER (System/Procedural)\n\n### **BTEQ (Batch Processing)**\n- **Request Count**: 1,957 requests\n- **Total CPU Time**: 3.17 seconds\n- **Total I/O Operations**: 19,923 logical I/O\n- **Total I/O Volume**: 0.64 GB logical I/O\n- **Physical I/O**: 277 operations, ~4.5 MB physical I/O\n- **Workload Type**: ETL/ELT (System/Procedural)\n- **Peak Usage**: June 8th at 5 PM\n\n### **NODE (System Process)**\n- **Request Count**: 2 requests\n- **Total CPU Time**: 0.024 seconds\n- **Total I/O Operations**: 20 logical I/O\n- **Total I/O Volume**: 0.0006 GB logical I/O\n- **User**: DBC (system user)\n- **Workload Type**: OTHER (System/Procedural)\n\n### **Key Insights:**\n1. **PYTHON3 dominates resource usage** - accounting for the vast majority of CPU, I/O, and request volume\n2. **Peak hours**: Generally between 5-7 PM on weekdays\n3. **Workload pattern**: Most activity is analytical queries (\"Answers\" complexity)\n4. **I/O efficiency**: Low physical I/O relative to logical I/O indicates good caching\n\nWould you like me to drill down into any specific application, time period, or workload type?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_databaseVersion_happy", + "case_type": "happy_path", + "description": "Get current Teradata software version", + "input": "What release of Teradata software is currently running on our system?", + "expected_tools": [ + { + "name": "dba_databaseVersion", + "params": {} + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_databaseVersion", + "params": {} + } + ], + "actual_output": "Your Teradata system is currently running:\n\n- **Version/Release**: 20.00.24.60\n- **Language Support Mode**: Standard\n\nThis is Teradata Database version 20.00, which is part of the Teradata Vantage platform. The build/maintenance level is 24.60.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_flowControl_happy", + "case_type": "happy_path", + "description": "System throttling and delay statistics for a date range", + "input": "Show me the system throttling and flow control delay statistics from January 15th 2024 through February 28th 2024.", + "expected_tools": [ + { + "name": "dba_flowControl", + "params": { + "start_date": "2024-01-15", + "end_date": "2024-02-28" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_flowControl", + "params": { + "start_date": "2024-01-15", + "end_date": "2024-02-28" + } + } + ], + "actual_output": "Based on the query results, **no flow control events or throttling delays were recorded** during the period from January 15th, 2024 through February 28th, 2024.\n\nThis indicates that:\n\n- The Teradata workload management system did not impose any flow control restrictions during this timeframe\n- There were no resource constraint conditions that required the system to throttle or delay queries\n- The system had sufficient capacity to handle the workload without triggering flow control mechanisms\n\n**What this means:**\nFlow control events typically occur when the system experiences resource pressure (CPU, I/O, memory, or AMP worker task exhaustion). The absence of flow control events suggests the system was operating within its capacity limits throughout this period.\n\nIf you're investigating performance issues during this timeframe, you may want to look at:\n- User-level query delays using the **dba_userDelay** tool to see if queries were waiting in queues\n- Resource consumption patterns using **dba_resusageSummary** to understand overall system utilization\n- Specific query performance from the query logs\n\nWould you like me to investigate any of these alternative performance metrics?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_featureUsage_happy", + "case_type": "happy_path", + "description": "Feature utilization metrics for a date range", + "input": "Show me how different Teradata features were being utilized by our users between January 15th, 2024 and March 31st, 2024.", + "expected_tools": [ + { + "name": "dba_featureUsage", + "params": { + "start_date": "2024-01-15", + "end_date": "2024-03-31" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_featureUsage", + "params": { + "start_date": "2024-01-15", + "end_date": "2024-03-31" + } + } + ], + "actual_output": "Based on the query results, **no Teradata feature usage data was found** for the period between January 15th, 2024 and March 31st, 2024.\n\nThis could mean:\n\n1. **No query logs available**: The DBC.DBQLogTbl (query logging table) may not contain data for this time period\n2. **Query logging not enabled**: Feature usage tracking requires query logging to be enabled on the Teradata system\n3. **Logs aged out**: Historical query logs may have been purged before this date range\n4. **No queries executed**: No user queries were run during this period (unlikely)\n\n**To investigate further, you could:**\n\n- Check what date ranges ARE available in the query logs\n- Verify that query logging is enabled on your Teradata system\n- Check with your DBA about log retention policies\n- Try a more recent date range to see if current data is being captured\n\nWould you like me to check what query log data is available or investigate a different time period?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_userDelay_happy", + "case_type": "happy_path", + "description": "How long users waited in queues for a date range", + "input": "Show me how long users were waiting in the query queue between March 1st 2024 and March 31st 2024.", + "expected_tools": [ + { + "name": "dba_userDelay", + "params": { + "start_date": "2024-03-01", + "end_date": "2024-03-31" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_userDelay", + "params": { + "start_date": "2024-03-01", + "end_date": "2024-03-31" + } + } + ], + "actual_output": "Based on the query results for March 1-31, 2024, **there were no queries that experienced any queue delays during this period**. \n\nThe query specifically looks for instances where `DelayTime > 0`, meaning queries that had to wait in the queue before execution. Since no records were returned, this indicates that:\n\n- All queries during March 2024 were able to execute immediately without waiting\n- The system had sufficient resources available to handle the workload\n- No workload management throttling or queueing occurred during this time period\n\nThis is generally a positive indicator of system performance, suggesting that the Teradata system had adequate capacity to handle the query workload during March 2024 without requiring users to wait in queues.\n\nIf you'd like to:\n- Check a different time period that might have had more activity\n- Look at system-level flow control events (which would show throttling at the workload management level)\n- Investigate resource usage patterns during this period\n\nJust let me know!", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_sessionInfo_happy", + "case_type": "happy_path", + "description": "Active sessions for a specific user", + "input": "Show me what active connections the database user SALES_ADMIN currently has open in Teradata.", + "expected_tools": [ + { + "name": "dba_sessionInfo", + "params": { + "user_name": "SALES_ADMIN" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_sessionInfo", + "params": { + "user_name": "SALES_ADMIN" + } + } + ], + "actual_output": "The query returned no results, which means the database user **SALES_ADMIN** currently has **no active connections** open in Teradata.\n\nThis could mean:\n- The user is not currently logged in\n- All sessions for this user have been closed\n- The user account exists but has no active sessions at this moment\n\nIf you were expecting to see active sessions, you may want to verify:\n- The exact username spelling (it's case-insensitive in the query)\n- Whether the user has recently disconnected\n- If there are any connection issues preventing login", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_systemSpace_happy", + "case_type": "happy_path", + "description": "Total disk consumption across all databases", + "input": "How much disk capacity is currently being consumed across all databases in our data warehouse?", + "expected_tools": [ + { + "name": "dba_systemSpace", + "params": {} + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_systemSpace", + "params": {} + } + ], + "actual_output": "## Data Warehouse Disk Capacity Summary\n\nHere's the current disk space consumption across all databases in your data warehouse:\n\n| Metric | Value |\n|--------|-------|\n| **Space Used** | 0 GB |\n| **Space Allocated** | 92 GB |\n| **Percentage Used** | 0% |\n| **Free Space** | 92 GB |\n\n### Key Findings:\n\n- **Total Allocated Capacity**: 92 GB has been allocated across all databases\n- **Current Consumption**: Effectively 0 GB is currently being used (less than 1 GB)\n- **Available Space**: 92 GB remains available for use\n- **Utilization Rate**: 0% - your data warehouse is essentially empty or has minimal data\n\nThis indicates that your data warehouse has plenty of available capacity and is currently consuming very little disk space. If you'd like to see which specific databases or tables are using space, I can drill down into:\n- Space usage by individual database using `dba_databaseSpace`\n- Space usage by tables within a specific database using `dba_tableSpace`", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_tableSpace_ambiguous_vs_databaseSpace", + "case_type": "ambiguous_selection", + "description": "Ranking tables by size inside one database should prefer dba_tableSpace, not dba_databaseSpace", + "input": "Which tables in demo_user are using the most disk space?", + "expected_tools": [ + { + "name": "dba_tableSpace", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_tableSpace", + "params": { + "database_name": "demo_user" + } + } + ], + "actual_output": "Here are the tables in the **demo_user** database ranked by disk space usage:\n\n| **Table Name** | **Current Space (bytes)** | **Peak Space (bytes)** | **Skew %** |\n|---|---|---|---|\n| **get_data** | 294,912 | 294,912 | 35.71% |\n| **remove_data** | 262,144 | 262,144 | 33.33% |\n| **evals_orders** | 196,608 | 196,608 | 0.0% |\n| **space_report** | 196,608 | 196,608 | 25.0% |\n| **evals_employees** | 196,608 | 196,608 | 0.0% |\n\n**Key Findings:**\n- **Total database space:** 1,146,880 bytes (~1.09 MB)\n- **Largest table:** `get_data` uses 294,912 bytes (~288 KB), representing about 26% of the database\n- **Second largest:** `remove_data` uses 262,144 bytes (~256 KB), about 23% of the database\n- **Data skew:** Both `get_data` (35.71%) and `remove_data` (33.33%) show significant data skew across AMPs, while `evals_orders` and `evals_employees` have perfect distribution (0% skew)\n\nThe top two tables (`get_data` and `remove_data`) account for nearly half of the database's disk usage.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_databaseSpace_ambiguous_vs_systemSpace", + "case_type": "ambiguous_selection", + "description": "Storage for one named database should prefer dba_databaseSpace, not dba_systemSpace", + "input": "How much total space is the demo_user database using?", + "expected_tools": [ + { + "name": "dba_databaseSpace", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_databaseSpace", + "params": { + "database_name": "demo_user" + } + } + ], + "actual_output": "The **demo_user** database has the following disk space usage:\n\n- **Space Allocated**: 78.0 GB\n- **Space Used**: 0.0 GB\n- **Free Space**: 78.0 GB\n- **Percent Used**: 0.0%\n\nThe database currently has 78 GB of space allocated to it but is not using any of that space yet (0.0 GB used). This means the database exists but either has no tables in it or the tables are empty.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_systemSpace_ambiguous_vs_databaseSpace", + "case_type": "ambiguous_selection", + "description": "Warehouse-wide storage across all databases should prefer dba_systemSpace, not dba_databaseSpace", + "input": "What is the total disk usage across the entire Teradata system?", + "expected_tools": [ + { + "name": "dba_systemSpace", + "params": {} + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_systemSpace", + "params": {} + } + ], + "actual_output": "Based on the system-wide disk space analysis, here's the total disk usage across your entire Teradata system:\n\n## System Disk Space Summary\n\n- **Total Space Allocated**: 92 GB\n- **Space Currently Used**: 0 GB\n- **Percentage Used**: 0%\n- **Free Space Available**: 92 GB\n\nThe system currently has 92 GB of total allocated disk space with essentially no space being used at this time. This indicates either a new system or one that has been recently cleaned/reset with minimal data loaded.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_tableSqlList_ambiguous_vs_userSqlList", + "case_type": "ambiguous_selection", + "description": "Queries run against a table should prefer dba_tableSqlList, not dba_userSqlList", + "input": "What SQL has been executed against evals_orders in the past week?", + "expected_tools": [ + { + "name": "dba_tableSqlList", + "params": { + "table_name": "evals_orders", + "no_days": 7 + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_tableSqlList", + "params": { + "table_name": "evals_orders", + "no_days": 7 + } + } + ], + "actual_output": "Based on the query log for the `evals_orders` table over the past week, here are the SQL statements that have been executed:\n\n## Summary of SQL Activity\n\n**User:** DEMO_USER \n**Time Period:** Past 7 days (mostly from June 23, 2026)\n\n### Main Query Types:\n\n1. **Table Preview Queries:**\n - `SELECT TOP 5 * FROM demo_user.evals_orders`\n - Multiple executions to sample the table data\n\n2. **Filtered Data Retrieval:**\n - `SELECT * FROM demo_user.evals_orders WHERE amount > 500`\n - Executed multiple times to retrieve orders with amounts greater than 500\n\n3. **Count Queries:**\n - `SELECT COUNT(*) FROM demo_user.evals_orders`\n - Used to get total row counts\n\n4. **Table Metadata:**\n - `SHOW TABLE demo_user.evals_orders`\n - Multiple executions to view table structure\n\n5. **Table Affinity Analysis:**\n - Complex query joining `DBC.DBQLObjTbl` and `DBC.DBQLogTbl` to analyze which tables are queried together with `evals_orders`\n - This appears to be an internal system query for understanding table relationships\n\n### Key Observations:\n\n- All queries were executed by the **DEMO_USER** account\n- Most activity occurred on **June 23, 2026** between 20:38 and 21:17\n- The queries are primarily **read operations** (SELECT statements)\n- No INSERT, UPDATE, DELETE, or DDL operations were detected\n- Common patterns include data sampling, filtering, and metadata inspection\n\nThe table appears to be used mainly for data exploration and analysis purposes during this period.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_userSqlList_ambiguous_vs_tableSqlList", + "case_type": "ambiguous_selection", + "description": "Queries run by a specific user should prefer dba_userSqlList, not dba_tableSqlList", + "input": "List all SQL statements that user ANALYTICS_USER has run over the past 7 days.", + "expected_tools": [ + { + "name": "dba_userSqlList", + "params": { + "user_name": "ANALYTICS_USER", + "no_days": 7 + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_userSqlList", + "params": { + "user_name": "ANALYTICS_USER", + "no_days": 7 + } + } + ], + "actual_output": "The query returned **no SQL statements** for user ANALYTICS_USER over the past 7 days.\n\nThis could mean:\n1. **The user has not run any queries** in the past 7 days\n2. **Query logging is not enabled** or not capturing data for this user\n3. **The user name may be incorrect** - Teradata user names are case-sensitive\n4. **Query log data may have been purged** or is not being retained\n\n**Suggestions:**\n- Verify the exact user name (check capitalization)\n- Check if query logging (DBQL) is enabled on the system\n- Try expanding the time range beyond 7 days to see if historical data exists\n- Verify that the user actually has been running queries during this period\n\nWould you like me to:\n- Check with a different user name?\n- Extend the time range (e.g., 30 days)?\n- List all active users who have run queries recently?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_tableUsageImpact_ambiguous_vs_resusageSummary", + "case_type": "ambiguous_selection", + "description": "Heavy users of a specific database should prefer dba_tableUsageImpact, not dba_resusageSummary", + "input": "Who is generating the heaviest workload against demo_user?", + "expected_tools": [ + { + "name": "dba_tableUsageImpact", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_tableUsageImpact", + "params": { + "database_name": "demo_user" + } + } + ], + "actual_output": "The query returned no results, which means there's no recorded workload activity against the **demo_user** database in the DBQL (Database Query Log) tables.\n\nThis could happen for several reasons:\n\n1. **DBQL logging may not be enabled** for queries against this database\n2. **No recent query activity** has occurred against demo_user tables\n3. **The database may be empty** or not actively used\n4. **DBQL retention period** may have expired and historical data has been purged\n\nTo investigate further, I can:\n- Check if the demo_user database exists and contains tables\n- Look at system-wide resource usage to see if there's activity elsewhere\n- Check session information to see if there are active connections\n\nWould you like me to check if the demo_user database exists and what tables it contains?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_resusageSummary_ambiguous_vs_tableUsageImpact", + "case_type": "ambiguous_selection", + "description": "System-wide CPU/memory/I/O by application should prefer dba_resusageSummary, not dba_tableUsageImpact", + "input": "Break down CPU, memory, and I/O consumption by application over the last 30 days.", + "expected_tools": [ + { + "name": "dba_resusageSummary", + "params": { + "no_days": 30 + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_resusageSummary", + "params": { + "no_days": 30 + } + } + ], + "actual_output": "Based on the resource usage data over the last 30 days, here's the breakdown of CPU, memory, and I/O consumption by application:\n\n## **Resource Consumption by Application (Last 30 Days)**\n\n### **Primary Applications Identified:**\n\n#### **1. PYTHON3 (Dominant Application)**\n- **Request Count:** 577 total requests across various workloads\n- **CPU Usage:** \n - Peak session: 24.20 seconds (AMPCPUTime) on June 19\n - Average range: 1.40 - 12.32 seconds per session\n- **I/O Consumption:**\n - Peak logical I/O: **85,309 operations** (3.71 GB)\n - Typical range: 7,547 - 85,309 I/O operations\n - Physical I/O: Minimal (0 - 71 operations), indicating good caching\n- **Workload Types:**\n - Query/Answers (primary)\n - ETL/ELT\n - System/Procedural\n\n#### **2. BTEQ (Batch Processing)**\n- **Request Count:** 1,957 requests\n- **CPU Usage:** 3.17 seconds (AMPCPUTime)\n- **I/O Consumption:**\n - Logical I/O: 19,923 operations (0.64 GB)\n - Physical I/O: 277 operations (4.53 MB)\n- **Workload Type:** ETL/ELT System/Procedural\n\n#### **3. NODE (System Operations)**\n- **Request Count:** 2 requests (minimal activity)\n- **CPU Usage:** 0.024 seconds\n- **I/O Consumption:** 20 operations (0.56 MB)\n- **Workload Type:** System/Procedural\n\n### **Key Insights:**\n\n1. **PYTHON3 dominates resource consumption** - responsible for the majority of query workloads and highest CPU/I/O usage\n2. **Excellent cache efficiency** - Very low physical I/O compared to logical I/O indicates data is mostly served from cache\n3. **Peak activity periods:**\n - Monday evenings (18:00-19:00)\n - Friday early morning (01:00)\n - Tuesday evening (19:00)\n4. **Workload complexity:** Most queries classified as \"Answers\" complexity with some ETL/ELT procedural work\n\n**Note:** The data shows Total Server Byte Count as 0 across all entries, indicating memory metrics may not be captured in this particular logging configuration. CPU and I/O metrics are fully available and detailed above.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_tableUsageImpact_ambiguous_vs_sessionInfo", + "case_type": "ambiguous_selection", + "description": "Historical resource consumption by users in a database should prefer dba_tableUsageImpact, not dba_sessionInfo", + "input": "Which users have been consuming the most CPU and query resources in demo_user over time?", + "expected_tools": [ + { + "name": "dba_tableUsageImpact", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_tableUsageImpact", + "params": { + "database_name": "demo_user" + } + }, + { + "name": "dba_resusageSummary", + "params": { + "no_days": 30 + } + } + ], + "actual_output": "Perfect! Based on the resource usage data over the past 30 days, here's what I found about CPU and query resource consumption:\n\n## **CPU and Query Resource Consumption Summary**\n\n### **Top User: DEMO_USER**\n\nThe **DEMO_USER** account has been the dominant consumer of system resources. Here are the key findings:\n\n### **Highest Activity Periods:**\n\n1. **June 19 at 1:00 AM (Friday)** - Peak CPU usage\n - **24.20 CPU seconds** across 147 queries\n - 3.71 GB of I/O processed\n - Workload type: QUERY (Answers complexity)\n\n2. **June 15 at 6:00 PM (Monday)** - High volume\n - **12.32 CPU seconds** across 366 queries\n - 3.23 GB of I/O processed\n - Application: PYTHON3\n\n3. **June 23 at 7:00 PM (Tuesday)**\n - **5.73 CPU seconds** across 64 queries\n - 1.97 GB of I/O processed\n\n### **Workload Breakdown:**\n\n- **Query Workload (Answers)**: Most CPU-intensive, typically running analytical queries via PYTHON3\n- **ETL/ELT Operations**: Lower CPU but involved in data loading (using BTEQ on June 8)\n- **System/Procedural**: Administrative tasks with minimal resource consumption\n\n### **Secondary User:**\n- **DBC** (system account): Minimal usage - 0.024 CPU seconds on June 8\n\n### **Key Observations:**\n\n1. **DEMO_USER is the sole significant user** consuming resources in this system\n2. **Python-based applications** are the primary access method\n3. **Peak activity occurs during evening hours** (6-7 PM) and early morning (1 AM)\n4. **Monday and Friday** show the highest activity levels\n5. Total query count ranges from **5 to 366 queries** per session\n\nWould you like me to drill deeper into specific time periods or analyze the SQL queries being executed by DEMO_USER?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_sessionInfo_ambiguous_vs_tableUsageImpact", + "case_type": "ambiguous_selection", + "description": "Open sessions for a specific user should prefer dba_sessionInfo, not dba_tableUsageImpact", + "input": "What active sessions does user SALES_ADMIN currently have open?", + "expected_tools": [ + { + "name": "dba_sessionInfo", + "params": { + "user_name": "SALES_ADMIN" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_sessionInfo", + "params": { + "user_name": "SALES_ADMIN" + } + } + ], + "actual_output": "The user **SALES_ADMIN** currently has **no active sessions** open on the Teradata system.\n\nThis means:\n- The user is not currently logged in\n- There are no open connections or database sessions associated with this user account\n- No queries or operations are being executed under this user's credentials at this time\n\nIf you were expecting to see active sessions, the user may have logged out, their sessions may have timed out, or they simply aren't connected at the moment.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_userDelay_ambiguous_vs_flowControl", + "case_type": "ambiguous_selection", + "description": "How long users waited in queues should prefer dba_userDelay, not dba_flowControl", + "input": "How long were users waiting in the query queue between March 1 and March 31, 2024?", + "expected_tools": [ + { + "name": "dba_userDelay", + "params": { + "start_date": "2024-03-01", + "end_date": "2024-03-31" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_userDelay", + "params": { + "start_date": "2024-03-01", + "end_date": "2024-03-31" + } + } + ], + "actual_output": "Based on the query results, **there were no user delays in the query queue between March 1 and March 31, 2024**. \n\nThe system found zero instances where users had to wait (DelayTime > 0) before their queries started executing during this period. This indicates that:\n\n- **No queries experienced queue delays** - All queries were able to start executing immediately\n- **The system had sufficient capacity** - There were no resource constraints causing queries to wait\n- **Workload management was effective** - No throttling or queuing was necessary\n\nThis is a positive indicator of system performance, suggesting that the Teradata system had adequate resources to handle the workload during March 2024 without requiring users to wait in the query queue.\n\nIf you'd like to check:\n- System-wide throttling and flow control events (even without user delays), I can use the `dba_flowControl` tool\n- Resource usage patterns during this period to understand overall system load\n- A different time period to see if delays occurred at other times\n\nWould you like me to investigate any of these aspects?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_flowControl_ambiguous_vs_userDelay", + "case_type": "ambiguous_selection", + "description": "System throttling and flow-control statistics should prefer dba_flowControl, not dba_userDelay", + "input": "Show throttling and flow-control delay statistics from January 15 through February 28, 2024.", + "expected_tools": [ + { + "name": "dba_flowControl", + "params": { + "start_date": "2024-01-15", + "end_date": "2024-02-28" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_flowControl", + "params": { + "start_date": "2024-01-15", + "end_date": "2024-02-28" + } + } + ], + "actual_output": "## Throttling and Flow-Control Statistics\n**Period: January 15 - February 28, 2024**\n\n**No flow control events were recorded during this period.**\n\nThe query successfully executed against the Teradata workload management tables (DBC.RESUSAGESAWT, DBC.RESUSAGESVPR, DBC.RESUSAGESPMA), but returned zero rows. This indicates that:\n\n- **No throttling occurred**: The system did not impose any workload management flow control restrictions during this timeframe\n- **No resource-based delays**: Queries were not delayed due to CPU, memory, or I/O resource constraints\n- **Healthy system state**: The absence of flow control events suggests the system had adequate capacity to handle the workload without throttling\n\n**What the report would have shown (if events existed):**\n- Date and time of each flow control event\n- Duration of flow control delays\n- FlowControl% - percentage of time queries were throttled\n- CPU utilization metrics (execution, service, I/O wait, idle percentages)\n- Correlation between CPU pressure and throttling events\n\nThis is generally a positive indicator of system health during the specified period.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_tableSpace_missing_db", + "case_type": "missing_parameter", + "description": "Agent must ask which database to list biggest tables for", + "input": "Show me the biggest tables", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you see the biggest tables! However, I need to know which database you'd like to check. \n\nCould you please specify which database you want to see the largest tables for?\n\nFor example:\n- A specific database name (e.g., \"SALES_DB\", \"DEV01_StGeo_STD_T\")\n- Or if you'd like to see system-wide space usage across all databases", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_tableSqlList_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask which table to list queries for", + "input": "Show me the queries from the last 14 days", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to show you the queries from the last 14 days, but I need a bit more information to give you the most relevant results.\n\nCould you please clarify what type of query history you're looking for?\n\n1. **Queries run by a specific user** - If you want to see queries executed by a particular user account, I can use the user SQL history tool. Please provide the username.\n\n2. **Queries run against a specific table** - If you want to see queries that accessed a particular table, I can use the table SQL history tool. Please provide the table name.\n\n3. **System-wide query activity** - If you want broader system metrics like resource usage, workload patterns, or user activity across the system, I can provide that as well.\n\nWhich would you prefer, and what specific details should I filter by?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_userSqlList_missing_user", + "case_type": "missing_parameter", + "description": "Agent must ask which user to list queries for", + "input": "Show me the queries a user executed over the past 14 days", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you retrieve the SQL queries executed by a user over the past 14 days. However, I need to know which specific user account you'd like to analyze.\n\nCould you please provide the username (Teradata user account name) you want to investigate?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_databaseSpace_missing_db", + "case_type": "missing_parameter", + "description": "Agent must ask which database to get storage for", + "input": "How much space is a specific database using?", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I can help you find out how much space a specific database is using. However, I need to know which database you'd like to check.\n\nCould you please provide the name of the database you want to analyze?\n\nOnce you give me the database name, I'll use the `dba_databaseSpace` tool to retrieve its disk space allocation and usage information.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_tableUsageImpact_missing_db", + "case_type": "missing_parameter", + "description": "Agent must ask which database to evaluate usage impact for", + "input": "Which users are hitting a particular database the most?", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you identify which users are most actively querying a particular database. However, I need to know which database you want to analyze.\n\n**Could you please specify the database name?**\n\nOnce you provide the database name, I'll use the `dba_tableUsageImpact` tool to show you:\n- Which users are generating the most query activity\n- Which tables are being accessed most frequently\n- Resource consumption patterns (CPU, IO, etc.)\n- Query counts and activity metrics\n\nFor example, if you want to analyze a database called \"SALES_DB\" or \"CUSTOMER_DATA\", just let me know the name and I'll retrieve the usage statistics for you.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "dba_databaseSpace_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which database, then calls dba_databaseSpace after user clarifies", + "input": "Turn 1: How much space is a specific database using? | Turn 2: How much disk space is the demo_user database using?", + "expected_tools": [ + { + "name": "dba_databaseSpace", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_databaseSpace", + "params": { + "database_name": "demo_user" + } + } + ], + "actual_output": "The **demo_user** database has the following disk space usage:\n\n- **Space Allocated**: 78.0 GB\n- **Space Used**: 0.0 GB\n- **Free Space**: 78.0 GB\n- **Percent Used**: 0.0%\n\nThe database currently has 78 GB of space allocated to it but is not using any of that space at the moment. This suggests the database either has no tables/data yet, or the tables are empty.", + "metric_reasons": [], + "recommendation": null, + "turn_details": [ + { + "turn": 1, + "input": "How much space is a specific database using?", + "mode": "clarification", + "passed": true, + "actual_tools": [] + }, + { + "turn": 2, + "input": "How much disk space is the demo_user database using?", + "mode": "tool", + "passed": true, + "actual_tools": [ + { + "name": "dba_databaseSpace", + "params": { + "database_name": "demo_user" + } + } + ] + } + ] + }, + { + "case_id": "dba_tableSqlList_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which table, then calls dba_tableSqlList after user clarifies", + "input": "Turn 1: Show me the queries from the last 14 days | Turn 2: List SQL run against evals_orders over the past 14 days", + "expected_tools": [ + { + "name": "dba_tableSqlList", + "params": { + "table_name": "evals_orders", + "no_days": 14 + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_tableSqlList", + "params": { + "table_name": "evals_orders", + "no_days": 14 + } + } + ], + "actual_output": "Here are the SQL queries that have been run against the **evals_orders** table over the past 14 days:\n\n## Query Summary\n\nAll queries were executed by **DEMO_USER** on **June 23, 2026**. Here are the distinct types of queries found:\n\n### 1. **Data Exploration Queries**\n```sql\nSELECT TOP 5 * FROM demo_user.evals_orders\n```\n- Simple preview queries to see sample data\n\n### 2. **Filtered Queries**\n```sql\nSELECT * FROM demo_user.evals_orders WHERE amount > 500\n```\n- Filtering orders where amount exceeds 500\n\n### 3. **Count Queries**\n```sql\nSELECT COUNT(*) FROM demo_user.evals_orders\n```\n- Getting total row counts\n\n### 4. **Schema Inspection**\n```sql\nSHOW TABLE demo_user.evals_orders\n```\n- Checking table structure and metadata\n\n### 5. **Complex Analytics Query**\n```sql\nLOCKING ROW FOR ACCESS\nSELECT\n TRIM(QTU2.DatabaseName) AS \"DatabaseName\",\n TRIM(QTU2.TableName) AS \"TableName\",\n COUNT(DISTINCT QTU1.QueryID) AS \"QueryCount\",\n (current_timestamp - min(QTU2.CollectTimeStamp)) day(4) AS \"FirstQueryDaysAgo\",\n (current_timestamp - max(QTU2.CollectTimeStamp)) day(4) AS \"LastQueryDaysAgo\"\nFROM (... complex join logic ...)\n```\n- Advanced query analyzing table affinity and usage patterns (likely from the system's own metadata tracking)\n\n## Activity Pattern\n- **Most Recent Activity**: June 23, 2026 at 21:17 (9:17 PM)\n- **Primary User**: DEMO_USER\n- **Query Types**: Mix of exploratory queries, basic filtering, and metadata inspection\n\nThe queries show typical exploratory and analytical activity, with users examining the data, checking counts, and filtering based on the `amount` column.", + "metric_reasons": [], + "recommendation": null, + "turn_details": [ + { + "turn": 1, + "input": "Show me the queries from the last 14 days", + "mode": "clarification", + "passed": true, + "actual_tools": [] + }, + { + "turn": 2, + "input": "List SQL run against evals_orders over the past 14 days", + "mode": "tool", + "passed": true, + "actual_tools": [ + { + "name": "dba_tableSqlList", + "params": { + "table_name": "evals_orders", + "no_days": 14 + } + } + ] + } + ] + }, + { + "case_id": "dba_userSqlList_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which user, then calls dba_userSqlList after user clarifies", + "input": "Turn 1: Show me the queries a user executed over the past 14 days | Turn 2: List all SQL that user ANALYTICS_USER has run over the last 14 days", + "expected_tools": [ + { + "name": "dba_userSqlList", + "params": { + "user_name": "ANALYTICS_USER", + "no_days": 14 + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "dba_userSqlList", + "params": { + "user_name": "ANALYTICS_USER", + "no_days": 14 + } + } + ], + "actual_output": "The query completed successfully, but **no SQL statements were found** for user **ANALYTICS_USER** over the last 14 days.\n\nThis could mean:\n- The user hasn't executed any queries in the past 14 days\n- The user account name might be slightly different (case-sensitive or has extra spaces)\n- Query logging may not be enabled or may have been purged\n- The user might be running queries under a different username\n\n**Would you like me to:**\n1. Check if there are any active sessions for this user?\n2. Verify the exact username exists in the system?\n3. Try a different time period or different username?", + "metric_reasons": [], + "recommendation": null, + "turn_details": [ + { + "turn": 1, + "input": "Show me the queries a user executed over the past 14 days", + "mode": "clarification", + "passed": true, + "actual_tools": [] + }, + { + "turn": 2, + "input": "List all SQL that user ANALYTICS_USER has run over the last 14 days", + "mode": "tool", + "passed": true, + "actual_tools": [ + { + "name": "dba_userSqlList", + "params": { + "user_name": "ANALYTICS_USER", + "no_days": 14 + } + } + ] + } + ] + }, + { + "case_id": "dba_unknown_tool_missing", + "case_type": "missing_parameter", + "description": "User asks for something no DBA tool supports \u2014 agent should explain and ask for clarification", + "input": "Automatically redistribute my table data across AMPs to improve skew", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I understand you want to redistribute table data across AMPs to improve skew, but I need more information to help you effectively:\n\n## Required Information:\n\n1. **Which table** do you want to redistribute? (Please provide the full table name)\n2. **Which database** is it in?\n\n## Additional Context That Would Help:\n\n3. Do you know if the table currently has high skew? (Have you measured CPU skew or data distribution skew?)\n4. What is the current Primary Index (PI) of the table?\n5. Do you have a candidate column or set of columns in mind for a new Primary Index?\n\n## What I Can Do For You:\n\nOnce you provide the table and database name, I can:\n\n- **Analyze current skew patterns** using SQL optimization tools (check CPU skew, I/O skew)\n- **Examine the table's schema** and current Primary Index\n- **Identify high-cardinality columns** that might make good PI candidates\n- **Generate a redistribution strategy** (ALTER TABLE with new UPI or NUPI)\n- **Create the DDL** to redistribute the data\n\n## Note on Redistribution:\n\nRedistributing data typically involves:\n- Changing the Primary Index definition (which controls AMP distribution)\n- This triggers a full table redistribution across all AMPs\n- For large tables, this can be resource-intensive\n\n**Please provide the table name and database name so I can analyze and help you redistribute the data.**", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "plot_line_chart_happy", + "case_type": "happy_path", + "description": "Line chart of order amounts over time from evals_orders", + "input": "Show me a line chart of order amounts over time from demo_user.evals_orders using order_date on the x-axis and amount on the y-axis", + "expected_tools": [ + { + "name": "plot_line_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "order_date", + "columns": [ + "amount" + ] + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "plot_line_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "order_date", + "columns": "amount" + } + }, + { + "name": "base_readQuery", + "params": { + "sql": "SELECT order_date, CAST(amount AS FLOAT) as amount FROM demo_user.evals_orders ORDER BY order_date" + } + } + ], + "actual_output": "Perfect! Here's the data from your evals_orders table showing order amounts over time:\n\n## Order Amounts Over Time - Line Chart Data\n\n| Order Date | Amount |\n|------------|---------|\n| 2024-01-05 | $1,250.00 |\n| 2024-01-12 | -$200.00 |\n| 2024-02-03 | $875.50 |\n| 2024-02-17 | $3,400.00 |\n| 2024-03-01 | $560.00 |\n| 2024-03-15 | -$150.00 |\n| 2024-04-02 | $2,100.00 |\n| 2024-04-20 | $930.00 |\n| 2024-05-08 | $1,780.00 |\n| 2024-05-25 | $4,200.00 |\n\nThe data shows order amounts from January through May 2024. Notable observations:\n- **Highest order**: $4,200 on May 25, 2024\n- **Two negative amounts**: -$200 (Jan 12) and -$150 (Mar 15), which may represent returns or refunds\n- **Overall trend**: Generally positive with some fluctuation, peaking in late May\n\nThe line chart would show the progression of order amounts over this ~5-month period, with the negative values creating dips in January and March, and a strong upward movement toward the May peak.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "plot_pie_chart_happy", + "case_type": "happy_path", + "description": "Pie chart breaking down orders by product category", + "input": "Plot demo_user.evals_orders as a pie chart using product_category for the slice labels and amount for the values.", + "expected_tools": [ + { + "name": "plot_pie_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "product_category", + "column": "amount" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "plot_pie_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "product_category", + "column": "amount" + } + }, + { + "name": "base_readQuery", + "params": { + "sql": "SELECT product_category, CAST(SUM(amount) AS FLOAT) as amount FROM demo_user.evals_orders GROUP BY product_category" + } + } + ], + "actual_output": "I can see the data aggregated by product category. Here's the breakdown:\n\n**Amount by Product Category:**\n- **Electronics**: $6,430.00 (largest category)\n- **Sports**: $6,300.00\n- **Food**: $1,805.50\n- **Books**: $560.00\n- **Clothing**: -$350.00 (negative value, likely returns/refunds)\n\nUnfortunately, there appears to be a technical issue with the direct pie chart plotting function and decimal data types. The data shows that Electronics and Sports dominate the sales, together accounting for the vast majority of the total amount, while Clothing has a negative value which may represent returns or refunds.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "plot_polar_chart_happy", + "case_type": "happy_path", + "description": "Polar area chart of order amounts by product category", + "input": "Create a polar area chart showing order amounts by product_category from demo_user.evals_orders", + "expected_tools": [ + { + "name": "plot_polar_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "product_category", + "column": "amount" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "plot_polar_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "product_category", + "column": "order_amount" + } + }, + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + }, + { + "name": "plot_polar_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "product_category", + "column": "amount" + } + }, + { + "name": "base_readQuery", + "params": { + "sql": "SELECT product_category, CAST(SUM(amount) AS FLOAT) as total_amount FROM demo_user.evals_orders GROUP BY product_category", + "persist": true + } + }, + { + "name": "plot_polar_chart", + "params": { + "column": "total_amount", + "labels": "product_category", + "table_name": "vt_d842dc42_33bd_41" + } + } + ], + "actual_output": "", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "plot_radar_chart_happy", + "case_type": "happy_path", + "description": "Radar chart of salary by department in evals_employees", + "input": "Draw a spider chart from demo_user.evals_employees using department as the axis labels and salary as the values.", + "expected_tools": [ + { + "name": "plot_radar_chart", + "params": { + "table_name": "demo_user.evals_employees", + "labels": "department", + "columns": "salary" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "plot_radar_chart", + "params": { + "table_name": "demo_user.evals_employees", + "labels": "department", + "columns": "salary" + } + }, + { + "name": "base_readQuery", + "params": { + "sql": "SELECT department, CAST(AVG(salary) AS DECIMAL(18,2)) as avg_salary \nFROM demo_user.evals_employees \nGROUP BY department \nORDER BY department" + } + } + ], + "actual_output": "I see the data shows average salaries by department. Unfortunately, there appears to be a technical issue with the radar chart function handling the Decimal data type from the database. \n\nHere's the salary data by department that would be displayed in the spider chart:\n\n**Department Salary Overview:**\n- **Engineering**: $95,000 (highest)\n- **Finance**: $88,000\n- **Marketing**: $77,000\n- **Sales**: $70,750\n- **HR**: $68,000 (lowest)\n\nThe spider/radar chart would show these five departments as points around the perimeter, with salary values determining how far from the center each point extends. Engineering would have the longest spoke, followed by Finance, Marketing, Sales, and HR.\n\nThis visualization would help compare salary levels across departments at a glance, showing that Engineering has the highest average salaries while HR has the lowest among the five departments.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "plot_line_vs_radar_ambiguous", + "case_type": "ambiguous_selection", + "description": "Time-series data should use line chart, not radar", + "input": "Plot how order amounts change over time from demo_user.evals_orders using order_date on the x-axis", + "expected_tools": [ + { + "name": "plot_line_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "order_date", + "columns": [ + "amount" + ] + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "plot_line_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "order_date", + "columns": "order_amount" + } + }, + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + }, + { + "name": "plot_line_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "order_date", + "columns": "amount" + } + }, + { + "name": "base_readQuery", + "params": { + "sql": "SELECT order_date, CAST(amount AS FLOAT) as amount FROM demo_user.evals_orders ORDER BY order_date" + } + } + ], + "actual_output": "Perfect! I can see the data now. Here's a summary of how order amounts change over time from January to May 2024:\n\n**Order Amounts Over Time (demo_user.evals_orders)**\n\n| Order Date | Amount |\n|------------|---------|\n| 2024-01-05 | $1,250.00 |\n| 2024-01-12 | -$200.00 (return/refund) |\n| 2024-02-03 | $875.50 |\n| 2024-02-17 | $3,400.00 |\n| 2024-03-01 | $560.00 |\n| 2024-03-15 | -$150.00 (return/refund) |\n| 2024-04-02 | $2,100.00 |\n| 2024-04-20 | $930.00 |\n| 2024-05-08 | $1,780.00 |\n| 2024-05-25 | $4,200.00 |\n\n**Key Observations:**\n- The data spans from January to May 2024 with 10 orders\n- Order amounts range from -$200 to $4,200\n- There are two negative amounts (returns/refunds) in January and March\n- The highest order amount ($4,200) occurred on May 25th\n- February 17th also had a significant order at $3,400\n\nThe trend shows some volatility with occasional high-value orders and a couple of returns, with the highest activity value occurring at the end of May.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "plot_pie_vs_polar_ambiguous", + "case_type": "ambiguous_selection", + "description": "Proportion/share breakdown should use pie chart over polar", + "input": "Use a pie chart to show demo_user.evals_orders with product_category as labels and amount as the values.", + "expected_tools": [ + { + "name": "plot_pie_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "product_category", + "column": "amount" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "plot_pie_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "product_category", + "column": "amount" + } + }, + { + "name": "base_readQuery", + "params": { + "sql": "SELECT product_category, SUM(amount) as total_amount FROM demo_user.evals_orders GROUP BY product_category" + } + } + ], + "actual_output": "Based on the data from demo_user.evals_orders, here's the breakdown by product category:\n\n## Product Category Distribution by Amount\n\n| Product Category | Total Amount |\n|-----------------|--------------|\n| **Electronics** | $6,430.00 |\n| **Sports** | $6,300.00 |\n| **Food** | $1,805.50 |\n| **Books** | $560.00 |\n| **Clothing** | -$350.00 |\n\n**Total:** $14,745.50\n\n**Key Insights:**\n- **Electronics** and **Sports** are the top categories, each accounting for over $6,000 in sales\n- **Food** represents a moderate amount at $1,805.50\n- **Books** contributes $560.00\n- **Clothing** shows a negative value (-$350.00), which may indicate returns or refunds\n\nNote: The pie chart visualization encountered a technical issue with decimal number formatting, but the data summary above provides the complete picture of how amounts are distributed across product categories.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "plot_missing_data", + "case_type": "missing_parameter", + "description": "Chart request with no table or data provided \u2014 agent must ask for clarification", + "input": "Create a chart for me", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to create a chart for you! To do that, I need a bit more information:\n\n1. **What type of chart** would you like?\n - Line chart (for trends over time)\n - Pie chart (for proportions/breakdowns)\n - Polar chart (for circular category comparisons)\n - Radar/spider chart (for multi-dimensional comparisons)\n\n2. **What data** should be visualized?\n - Which table/database contains the data?\n - What columns should be displayed?\n - Any specific filtering or time period?\n\nFor example, you could say:\n- \"Create a line chart showing sales over time from the Sales table\"\n- \"Create a pie chart of revenue by region from the Revenue_Data table\"\n- \"Show me a radar chart comparing product metrics\"\n\nWhat would you like to visualize?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "plot_pie_chart_direct", + "case_type": "happy_path", + "description": "Direct pie chart from table \u2014 no pre-query needed since plot tools read tables directly", + "input": "Give me a pie chart of amount by product_category from demo_user.evals_orders", + "expected_tools": [ + { + "name": "plot_pie_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "product_category", + "column": "amount" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "plot_pie_chart", + "params": { + "table_name": "demo_user.evals_orders", + "labels": "product_category", + "column": "amount" + } + }, + { + "name": "base_readQuery", + "params": { + "sql": "SELECT product_category, SUM(amount) as total_amount FROM demo_user.evals_orders GROUP BY product_category" + } + } + ], + "actual_output": "Here's the pie chart breakdown of amount by product category from demo_user.evals_orders:\n\n**Amount by Product Category:**\n\n| Product Category | Total Amount |\n|-----------------|--------------|\n| **Electronics** | $6,430.00 (44.6%) |\n| **Sports** | $6,300.00 (43.7%) |\n| **Food** | $1,805.50 (12.5%) |\n| **Books** | $560.00 (3.9%) |\n| **Clothing** | -$350.00 (-2.4%) |\n\n**Key Insights:**\n- **Electronics** and **Sports** dominate, together accounting for 88.3% of total amount\n- **Clothing** shows a negative amount (-$350), which may indicate returns or refunds\n- Total amount across all categories: **$14,745.50**\n\nNote: The negative value in Clothing suggests there were returns or credit transactions that exceeded sales in that category.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "plot_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask which table to plot", + "input": "Create a line chart of sales over time", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you create a line chart of sales over time! However, I need a few more details to get started:\n\n1. **Table name**: Which table contains the sales data?\n2. **Database name**: Which database is this table in?\n3. **Date/Time column**: What is the name of the column that contains the date or time information?\n4. **Sales column**: What is the name of the column that contains the sales values?\n\nFor example, if you have a table called `sales_data` in database `analytics` with columns `sale_date` and `sales_amount`, I would use those.\n\nCould you provide these details so I can create the line chart for you?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "plot_unknown_tool_missing", + "case_type": "missing_parameter", + "description": "User asks for a non\u2011existent plot tool", + "input": "Show me the network topology as a diagram", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I understand you'd like to see a network topology diagram. However, I need more specific information to help you with this request:\n\n1. **What type of network topology are you referring to?**\n - Database dependency graph (tables, views, and their relationships)?\n - Data lineage network (upstream/downstream dependencies)?\n - System architecture diagram?\n\n2. **Which database or objects would you like to analyze?**\n - A specific database name?\n - Specific tables or views?\n - A pattern like '%WBC%' or '%SALES%'?\n\nThe available tools I have can help you visualize:\n\n- **Database dependency graphs**: Using graph analysis tools, I can show you:\n - Root objects (starting points with no dependencies)\n - Downstream dependencies (what depends on what)\n - Upstream dependencies (what objects depend on)\n - Circular dependencies (cycles)\n - Connected components (isolated groups)\n\n- **Table relationship networks**: Show which tables are commonly queried together (affinity analysis)\n\nCould you please clarify:\n- What database or schema you want to analyze?\n- What specific aspect of the topology you're interested in (dependencies, lineage, relationships)?\n\nOnce you provide this information, I can generate the appropriate network diagram for you.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_missingValues_happy", + "case_type": "happy_path", + "description": "Find columns with nulls in evals_orders (ship_date is nullable)", + "input": "Which columns in demo_user.evals_orders contain null or empty entries?", + "expected_tools": [ + { + "name": "qlty_missingValues", + "params": {} + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_missingValues", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "actual_output": "Based on the analysis of the `demo_user.evals_orders` table, here are the columns with NULL or empty entries:\n\n## Columns with Missing Values:\n\n**1. ship_date**\n- **NULL Count:** 3\n- **NULL Percentage:** 30.0%\n\nThis is the only column that contains NULL values in the table.\n\n## Columns with No Missing Values:\n\nThe following columns have no NULL or missing values:\n- `order_id`\n- `customer_name`\n- `order_date`\n- `product_category`\n- `amount`\n- `quantity`\n\nThe `ship_date` column has 3 NULL entries out of 10 total rows (30%), which could indicate orders that haven't been shipped yet or missing shipping information.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_missingValues_ambiguous_vs_rowsWithMissingValues", + "case_type": "ambiguous_selection", + "description": "Asking which columns have nulls (column-level summary) should prefer qlty_missingValues over qlty_rowsWithMissingValues", + "input": "Which fields in demo_user.evals_orders have missing data?", + "expected_tools": [ + { + "name": "qlty_missingValues", + "params": {} + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_missingValues", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "actual_output": "The `demo_user.evals_orders` table has missing data in **one field**:\n\n- **ship_date**: 3 null values (30.0% of records)\n\nAll other fields have complete data with no missing values:\n- order_id\n- quantity\n- amount\n- customer_name\n- order_date\n- product_category\n\nSo about 30% of the orders in this table are missing their shipping dates, which might indicate orders that haven't been shipped yet or incomplete data entry.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_negativeValues_happy", + "case_type": "happy_path", + "description": "Find negative amounts in evals_orders (some amounts are negative refunds)", + "input": "Show me which fields in demo_user.evals_orders contain numbers below zero", + "expected_tools": [ + { + "name": "qlty_negativeValues", + "params": {} + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_negativeValues", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "actual_output": "Based on the analysis of the `demo_user.evals_orders` table, here are the fields that contain numbers below zero:\n\n**Field with Negative Values:**\n- **amount** - Contains 2 records with negative values\n\n**Fields with No Negative Values:**\n- **quantity** - 0 negative values\n- **order_id** - 0 negative values\n\nThe other fields (ship_date, product_category, order_date, customer_name) are non-numeric fields, so negative value checking doesn't apply to them.\n\nThe `amount` field is the only column in your table that has negative numbers, with 2 instances of values below zero.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_distinctCategories_happy", + "case_type": "happy_path", + "description": "List unique product categories in evals_orders", + "input": "Show me all the unique values in the product_category column of demo_user.evals_orders", + "expected_tools": [ + { + "name": "qlty_distinctCategories", + "params": { + "column_name": "product_category" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_distinctCategories", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders", + "column_name": "product_category" + } + } + ], + "actual_output": "Perfect! Here are all the unique values in the **product_category** column of **demo_user.evals_orders**:\n\n| Product Category | Count of Records |\n|-----------------|------------------|\n| **Electronics** | 3 |\n| **Books** | 1 |\n| **Clothing** | 2 |\n| **Sports** | 2 |\n| **Food** | 2 |\n\nThere are **5 distinct product categories** in total, with Electronics being the most common (appearing 3 times) and Books being the least common (appearing once).", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_distinctCategories_missing_column", + "case_type": "missing_parameter", + "description": "Agent must ask which column when none is specified", + "input": "What distinct values exist in a column of demo_user.evals_orders?", + "expected_tools": [], + "passed": false, + "failure_stage": "deterministic", + "failure_detail": "expected no tool calls for missing_parameter case, got ['base_columnDescription']", + "actual_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "actual_output": "The `demo_user.evals_orders` table has the following columns:\n\n1. **product_category** (VARCHAR)\n2. **ship_date** (DATE)\n3. **customer_name** (VARCHAR)\n4. **order_id** (INTEGER)\n5. **quantity** (INTEGER)\n6. **amount** (DECIMAL)\n7. **order_date** (DATE)\n\nWhich column would you like to see the distinct values for?", + "metric_reasons": [], + "recommendation": "The prompt deliberately omits required information: \"What distinct values exist in a column of demo_user.evals_orders?\". The agent called `base_columnDescription` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response.", + "turn_details": null + }, + { + "case_id": "qlty_standardDeviation_happy", + "case_type": "happy_path", + "description": "Get mean and spread of salary in evals_employees", + "input": "What is the average and spread of salary values in demo_user.evals_employees?", + "expected_tools": [ + { + "name": "qlty_standardDeviation", + "params": { + "column_name": "salary" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_standardDeviation", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees", + "column_name": "salary" + } + } + ], + "actual_output": "Based on the analysis of the salary column in demo_user.evals_employees:\n\n**Average Salary:** $83,055.56\n\n**Standard Deviation (Spread):** $12,053.12\n\nThis means:\n- The typical salary in this dataset is around $83,056\n- Most salaries fall within approximately one standard deviation of the mean, which would be roughly between $71,002 and $95,109\n- The standard deviation of about $12,053 indicates moderate variability in salary values across the employees in this table", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_standardDeviation_ambiguous_vs_univariateStatistics", + "case_type": "ambiguous_selection", + "description": "Asking specifically for standard deviation should prefer qlty_standardDeviation over the broader qlty_univariateStatistics", + "input": "What is the standard deviation of the salary field in demo_user.evals_employees?", + "expected_tools": [ + { + "name": "qlty_standardDeviation", + "params": { + "column_name": "salary" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_standardDeviation", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees", + "column_name": "salary" + } + } + ], + "actual_output": "The standard deviation of the salary field in demo_user.evals_employees is **12,053.12**.\n\nAdditionally, the mean (average) salary is **83,055.56**.\n\nThis indicates that salaries in this table vary by approximately $12,053 from the average, which suggests a moderate spread in the salary distribution.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_columnSummary_happy", + "case_type": "happy_path", + "description": "Get a statistical summary of all columns in evals_employees", + "input": "Show me a statistical summary of every field in demo_user.evals_employees", + "expected_tools": [ + { + "name": "qlty_columnSummary", + "params": {} + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_columnSummary", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "actual_output": "Here's a statistical summary of every field in **demo_user.evals_employees**:\n\n## String/Character Fields\n\n| Column | Data Type | Non-Null Count | Null Count | Blank Count | Null % |\n|--------|-----------|----------------|------------|-------------|--------|\n| **name** | VARCHAR(100) | 10 | 0 | 0 | 0.0% |\n| **region** | VARCHAR(50) | 10 | 0 | 0 | 0.0% |\n| **department** | VARCHAR(50) | 10 | 0 | 0 | 0.0% |\n\n## Numeric Fields\n\n| Column | Data Type | Non-Null Count | Null Count | Zero Count | Positive Count | Negative Count | Null % |\n|--------|-----------|----------------|------------|------------|----------------|----------------|--------|\n| **employee_id** | INTEGER | 10 | 0 | 0 | 10 | 0 | 0.0% |\n| **manager_id** | INTEGER | 9 | 1 | 0 | 9 | 0 | 10.0% |\n| **salary** | DECIMAL(10,2) | 9 | 1 | 0 | 9 | 0 | 10.0% |\n\n## Date Fields\n\n| Column | Data Type | Non-Null Count | Null Count | Null % |\n|--------|-----------|----------------|------------|--------|\n| **hire_date** | DATE | 10 | 0 | 0.0% |\n\n### Key Findings:\n- **Total rows**: 10 employees\n- **Complete fields**: name, region, department, employee_id, and hire_date have 100% data completeness\n- **Missing data**: manager_id and salary each have 1 missing value (10% null rate)\n- **All numeric values are positive**: No negative or zero values in employee_id, manager_id, or salary\n- **No blank strings**: All text fields are properly populated", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_univariateStatistics_happy", + "case_type": "happy_path", + "description": "Get full univariate stats for salary in evals_employees", + "input": "Give me a complete statistical breakdown of the salary field in demo_user.evals_employees", + "expected_tools": [ + { + "name": "qlty_univariateStatistics", + "params": { + "column_name": "salary" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_univariateStatistics", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees", + "column_name": "salary" + } + } + ], + "actual_output": "## Complete Statistical Breakdown - Salary Field (demo_user.evals_employees)\n\n### **Central Tendency Measures**\n- **Mean**: $83,055.56\n- **Median**: $85,000.00\n- **Mode**: $68,000.00\n- **Geometric Mean**: $82,278.64\n- **Harmonic Mean**: $81,508.55\n- **Trimmed Mean**: $82,500.00\n\n### **Dispersion Measures**\n- **Standard Deviation**: $12,053.12\n- **Variance**: $145,277,777.78\n- **Coefficient of Variation**: 14.51%\n- **Range**: $34,000.00\n- **Interquartile Range**: $19,000.00\n- **Standard Error**: $4,017.71\n\n### **Range & Extremes**\n- **Minimum**: $68,000.00\n- **Maximum**: $102,000.00\n- **Bottom 5 Values**: $68,000, $69,500, $72,000, $77,000, $85,000\n- **Top 5 Values**: $102,000, $95,000, $91,000, $88,000, $85,000\n\n### **Percentiles**\n- **1st Percentile**: $68,120.00\n- **5th Percentile**: $68,600.00\n- **10th Percentile**: $69,200.00\n- **25th Percentile (Q1)**: $72,000.00\n- **50th Percentile (Q2/Median)**: $85,000.00\n- **75th Percentile (Q3)**: $91,000.00\n- **90th Percentile**: $96,400.00\n- **95th Percentile**: $99,200.00\n- **99th Percentile**: $101,440.00\n\n### **Distribution Shape**\n- **Skewness**: 0.152 (slightly right-skewed, nearly symmetric)\n- **Kurtosis**: -1.333 (platykurtic - flatter than normal distribution)\n\n### **Data Quality**\n- **Count**: 9 salary values\n- **Null Count**: 1 missing value\n- **Unique Values**: 9\n- **Zero Values**: 0\n- **Negative Values**: 0\n- **Positive Values**: 9\n\n### **Aggregate Measures**\n- **Sum**: $747,500.00\n- **Corrected Sum of Squares**: $1,162,222,222.22\n- **Uncorrected Sum of Squares**: $63,246,250,000.00\n\n### **Key Insights**\n\u2713 The salary distribution is fairly balanced with a mean of $83,055 and median of $85,000 \n\u2713 Low coefficient of variation (14.51%) indicates moderate consistency \n\u2713 Slight positive skew suggests a few higher salaries pulling the mean slightly up \n\u2713 The interquartile range of $19,000 shows reasonable mid-range spread \n\u2713 One employee has a missing salary value", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_univariateStatistics_ambiguous_vs_columnSummary", + "case_type": "ambiguous_selection", + "description": "Asking for stats on a specific column should prefer qlty_univariateStatistics over the table-level qlty_columnSummary", + "input": "Show me min, max, mean, and percentiles for the amount column in demo_user.evals_orders", + "expected_tools": [ + { + "name": "qlty_univariateStatistics", + "params": { + "column_name": "amount" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_univariateStatistics", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders", + "column_name": "amount" + } + } + ], + "actual_output": "Here are the key statistics for the **amount** column in **demo_user.evals_orders**:\n\n## Summary Statistics\n\n- **Minimum**: -200.0\n- **Maximum**: 4,200.0\n- **Mean**: 1,474.55\n- **Median**: 1,090.0\n\n## Percentiles\n\n| Percentile | Value |\n|------------|--------|\n| 1st | -195.5 |\n| 5th | -177.5 |\n| 10th | -155.0 |\n| 25th (Q1) | 638.9 |\n| 50th (Median) | 1,090.0 |\n| 75th (Q3) | 2,020.0 |\n| 90th | 3,480.0 |\n| 95th | 3,840.0 |\n| 99th | 4,128.0 |\n\n## Additional Insights\n\n- **Standard Deviation**: 1,437.8\n- **Count**: 10 records\n- **Range**: 4,400.0 (from -200 to 4,200)\n- **Negative Values**: 2 records have negative amounts\n- **Interquartile Range (IQR)**: 1,381.1\n\nThe data shows a right-skewed distribution (skewness = 0.82) with some negative values that appear to be returns or refunds, and most values clustering in the lower-to-mid range with a few high-value outliers.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_rowsWithMissingValues_happy", + "case_type": "happy_path", + "description": "Retrieve rows where ship_date is null in evals_orders", + "input": "Show me all records from demo_user.evals_orders where the ship_date is empty or null", + "expected_tools": [ + { + "name": "qlty_rowsWithMissingValues", + "params": { + "column_name": "ship_date" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_rowsWithMissingValues", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders", + "column_name": "ship_date" + } + } + ], + "actual_output": "I found **3 records** from demo_user.evals_orders where the ship_date is null:\n\n| Order ID | Customer Name | Order Date | Ship Date | Amount | Product Category | Quantity |\n|----------|---------------|------------|-----------|---------|------------------|----------|\n| 1009 | Iota Inc | 2024-05-08 | null | 1780.0 | Electronics | 1 |\n| 1006 | Zeta Partners | 2024-03-15 | null | -150.0 | Clothing | 1 |\n| 1003 | Gamma Inc | 2024-02-03 | null | 875.5 | Food | 5 |\n\nAll three of these orders have no ship date recorded, indicating they may not have been shipped yet or the shipping information wasn't entered into the system.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_rowsWithMissingValues_ambiguous_vs_missingValues", + "case_type": "ambiguous_selection", + "description": "Asking to retrieve the actual rows with missing data should prefer qlty_rowsWithMissingValues over qlty_missingValues", + "input": "Fetch the rows from demo_user.evals_orders that are missing a ship_date", + "expected_tools": [ + { + "name": "qlty_rowsWithMissingValues", + "params": { + "column_name": "ship_date" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_rowsWithMissingValues", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders", + "column_name": "ship_date" + } + } + ], + "actual_output": "I found **3 rows** in demo_user.evals_orders that are missing a ship_date:\n\n| order_id | customer_name | order_date | ship_date | amount | product_category | quantity |\n|----------|---------------|------------|-----------|---------|------------------|----------|\n| 1009 | Iota Inc | 2024-05-08 | NULL | 1780.0 | Electronics | 1 |\n| 1006 | Zeta Partners | 2024-03-15 | NULL | -150.0 | Clothing | 1 |\n| 1003 | Gamma Inc | 2024-02-03 | NULL | 875.5 | Food | 5 |\n\nThese are orders that have been placed but not yet shipped.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_multi_tool_summary_then_nulls", + "case_type": "multi_tool", + "description": "Get column summary then drill into rows with missing values", + "input": "First give me a column summary of demo_user.evals_orders, then show me the rows where ship_date is missing", + "expected_tools": [ + { + "name": "qlty_columnSummary", + "params": {} + }, + { + "name": "qlty_rowsWithMissingValues", + "params": { + "column_name": "ship_date" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "qlty_columnSummary", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + }, + { + "name": "qlty_rowsWithMissingValues", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders", + "column_name": "ship_date" + } + } + ], + "actual_output": "## Column Summary for demo_user.evals_orders\n\nThe table has **7 columns** with the following characteristics:\n\n| Column Name | Data Type | Non-Null Count | Null Count | Null % | Additional Info |\n|-------------|-----------|----------------|------------|--------|-----------------|\n| **order_id** | INTEGER | 10 | 0 | 0% | All positive values |\n| **customer_name** | VARCHAR(100) | 10 | 0 | 0% | No blanks |\n| **order_date** | DATE | 10 | 0 | 0% | Complete |\n| **ship_date** | DATE | 7 | **3** | **30%** | Missing for 3 orders |\n| **amount** | DECIMAL(10,2) | 10 | 0 | 0% | 8 positive, 2 negative |\n| **product_category** | VARCHAR(50) | 10 | 0 | 0% | No blanks |\n| **quantity** | INTEGER | 10 | 0 | 0% | All positive values |\n\n---\n\n## Rows Where ship_date is Missing\n\nThere are **3 orders** with missing ship dates:\n\n| Order ID | Customer Name | Order Date | Ship Date | Amount | Product Category | Quantity |\n|----------|---------------|------------|-----------|--------|------------------|----------|\n| **1009** | Iota Inc | 2024-05-08 | NULL | $1,780.00 | Electronics | 1 |\n| **1006** | Zeta Partners | 2024-03-15 | NULL | -$150.00 | Clothing | 1 |\n| **1003** | Gamma Inc | 2024-02-03 | NULL | $875.50 | Food | 5 |\n\n**Note:** Order 1006 (Zeta Partners) has a negative amount of -$150.00, which may indicate a return or credit.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "qlty_unknown_tool_missing", + "case_type": "missing_parameter", + "description": "User asks for a non\u2011existent quality tool", + "input": "Give me a data quality score for the dataset", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you calculate a data quality score for a dataset. However, I need you to provide some information first:\n\n1. **Database name** (if applicable, otherwise I can use the default)\n2. **Table name** - Which specific table/dataset do you want to analyze?\n\nOnce you provide the table name, I can run several data quality checks including:\n- **Missing values analysis** - identifying columns with NULL values\n- **Column summary statistics** - providing an overview of all columns\n- **Negative values check** - for numeric columns where negatives might be anomalous\n- **Univariate statistics** - detailed statistical profiles for numeric columns\n\nPlease share the table name (and database name if you know it), and I'll generate a comprehensive data quality assessment for you.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_userDbPermissions_ambiguous_vs_userRoles", + "case_type": "ambiguous_selection", + "description": "Asking about a user's access rights on databases should prefer sec_userDbPermissions, not sec_userRoles", + "input": "Which databases can the user 'bob' access and what can they do in each one?", + "expected_tools": [ + { + "name": "sec_userDbPermissions", + "params": { + "user_name": "bob" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_userDbPermissions", + "params": { + "user_name": "bob" + } + }, + { + "name": "sec_userRoles", + "params": { + "user_name": "bob" + } + } + ], + "actual_output": "Based on the results:\n\n## Summary for User 'bob':\n\n**Direct Database Permissions:** None found\n\n**Role Memberships:** None found\n\nThis indicates that user 'bob' either:\n1. Does not exist in the system\n2. Has no permissions or roles currently assigned\n3. May only have permissions on their own user database (if one exists with the same name)\n\nIn Teradata, users typically get access to databases through either:\n- **Direct grants** - specific permissions granted directly to the user account\n- **Role memberships** - permissions inherited from roles assigned to the user\n\nSince both queries returned no results, user 'bob' appears to have no database access permissions configured at this time. If 'bob' needs access to databases, a database administrator would need to either:\n- Grant direct permissions to 'bob' on specific databases/tables\n- Assign 'bob' to one or more roles that have the necessary permissions\n\nWould you like me to check if the user 'bob' exists in the system, or would you like to grant them specific permissions?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_userDbPermissions_missing_user", + "case_type": "missing_parameter", + "description": "User asks about permissions without naming a user", + "input": "Show me the database permissions", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you view database permissions! To show you the relevant permissions, I need a bit more information:\n\n**What type of permissions are you looking for?**\n\n1. **User's database permissions** - What databases and tables a specific user can access (SELECT, INSERT, UPDATE, DELETE, etc.)\n - Need: Username\n\n2. **Role's database permissions** - What access rights are granted to a specific role\n - Need: Role name\n\n3. **User's role assignments** - What roles are assigned to a specific user\n - Need: Username\n\nPlease let me know which type you'd like to see and provide the username or role name accordingly.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_rolePermissions_ambiguous_vs_userDbPermissions", + "case_type": "ambiguous_selection", + "description": "Asking about a role's access should use sec_rolePermissions, not sec_userDbPermissions", + "input": "What access rights does the 'analyst' role have across all databases?", + "expected_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ], + "actual_output": "The query returned no results, which means the 'analyst' role either:\n\n1. **Does not exist** in the system\n2. **Has no database-level permissions granted** to it\n3. **Has no members** (the query joins role members with role rights)\n\nThis could happen if:\n- The role name is different (case-sensitive or spelled differently)\n- The role exists but hasn't been granted any database access rights yet\n- The role has no users assigned to it\n\nWould you like me to:\n- Check if the role exists with a different spelling or case?\n- Look up what roles are available in the system?\n- Check permissions for a specific user instead?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_userRoles_ambiguous_vs_rolePermissions", + "case_type": "ambiguous_selection", + "description": "Listing roles assigned to a user should prefer sec_userRoles, not sec_rolePermissions", + "input": "Which roles are currently assigned to user 'diana'?", + "expected_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "diana" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "diana" + } + } + ], + "actual_output": "The user **'diana'** currently has **no roles assigned**.\n\nThe query checked the Teradata system catalogs for any role memberships associated with this user account, but no role assignments were found. This means 'diana' either:\n- Has no roles granted to them\n- Relies on direct database-level permissions instead of role-based access\n- May be a newly created user account\n\nIf you'd like to check what direct database permissions 'diana' has instead, I can run that query using the `sec_userDbPermissions` tool.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_rolePermissions_ambiguous_vs_userRoles", + "case_type": "ambiguous_selection", + "description": "Permissions granted to a named role should prefer sec_rolePermissions, not sec_userRoles", + "input": "What database access has the 'analyst' role been granted?", + "expected_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ], + "actual_output": "The 'analyst' role currently has **no database access permissions** granted to it. The query returned zero results, which means:\n\n- No direct database-level permissions (SELECT, INSERT, UPDATE, DELETE, etc.) have been granted to this role\n- No CREATE, DROP, or ALTER privileges on database objects\n- No administrative or system-level privileges\n\nThis could mean:\n1. The role exists but hasn't been granted any permissions yet\n2. The role doesn't exist in the system\n3. Access is granted through other means (e.g., individual user permissions rather than role-based permissions)\n\nIf you expected this role to have permissions, you may want to:\n- Check if the role name is spelled correctly\n- Verify if permissions are granted directly to users instead of through this role\n- Check if there are other roles that users should be assigned to instead", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_userRoles_ambiguous_vs_userDbPermissions", + "case_type": "ambiguous_selection", + "description": "Asking about a user's role assignments should prefer sec_userRoles even though permissions is related", + "input": "Tell me which roles have been assigned to user 'diana'", + "expected_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "diana" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "diana" + } + } + ], + "actual_output": "The query returned no results for user 'diana'. This means that either:\n\n1. **The user 'diana' does not exist** in the Teradata system, or\n2. **The user 'diana' has no roles assigned** to them\n\nUsers in Teradata can exist without having any roles assigned - they might only have direct database permissions instead. If you'd like, I can check:\n- User 'diana's direct database permissions using `sec_userDbPermissions`\n- Whether the user exists at all by checking session or user information\n\nWould you like me to investigate further?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_multi_tool_user_roles_then_role_permissions", + "case_type": "multi_tool", + "description": "Find a user's roles, then inspect a specific named role's permissions in one pass", + "input": "Show me what roles 'alice' has, then list the permissions for the 'analyst' role", + "expected_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "alice" + } + }, + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "alice" + } + }, + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ], + "actual_output": "Based on the query results:\n\n## Roles for user 'alice'\n**No roles found** - The user 'alice' does not have any roles assigned in the system.\n\n## Permissions for role 'analyst'\n**No permissions found** - The role 'analyst' either doesn't exist in the system or has no database-level permissions granted to it.\n\n**Possible reasons:**\n1. The user 'alice' may not exist in the system\n2. The role 'analyst' may not exist in the system\n3. User and role names may be case-sensitive - you might want to try different capitalizations (e.g., 'Alice', 'ALICE', 'Analyst', 'ANALYST')\n4. The user exists but simply has no roles assigned\n5. The role exists but has no permissions granted yet\n\nWould you like me to check with different name variations, or would you like to see a list of all available users or roles in the system?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_userDbPermissions_happy", + "case_type": "happy_path", + "description": "Show what databases a user can access and what they can do in each", + "input": "Show me what access rights jsmith has across all the different databases in the system.", + "expected_tools": [ + { + "name": "sec_userDbPermissions", + "params": { + "user_name": "jsmith" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_userDbPermissions", + "params": { + "user_name": "jsmith" + } + } + ], + "actual_output": "The query returned no results for user 'jsmith'. This could mean one of the following:\n\n1. **The user doesn't exist** in the system\n2. **The user has no direct database permissions** granted (they may only have access through role memberships)\n3. **The username might be different** (case-sensitive or different format)\n\nWould you like me to:\n- **Check what roles are assigned to jsmith** (using `sec_userRoles`)? Users often receive their permissions through role memberships rather than direct grants.\n- **Search for similar usernames** if the exact username might be different?\n\nLet me know how you'd like to proceed!", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_rolePermissions_happy", + "case_type": "happy_path", + "description": "Show what permissions a named role has", + "input": "What access rights are granted to the FinanceReporter role?", + "expected_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "FinanceReporter" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "FinanceReporter" + } + } + ], + "actual_output": "The query returned no results for the **FinanceReporter** role. This could mean one of the following:\n\n1. **The role does not exist** in the Teradata system\n2. **The role has no database-level permissions granted** to it\n3. **The role name might be different** (case-sensitive or spelled differently)\n\nTo help troubleshoot, you might want to:\n- Verify the exact role name exists in the system\n- Check if the role has been granted any permissions at all\n- Ensure you have the necessary privileges to view role permissions\n\nWould you like me to help verify if this role exists or check permissions for a different role name?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_userRoles_happy", + "case_type": "happy_path", + "description": "List the roles assigned to a user account", + "input": "Which roles are assigned to the database account jsmith?", + "expected_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "jsmith" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "jsmith" + } + } + ], + "actual_output": "The database account **jsmith** currently has **no roles** assigned to it. The query returned zero results, indicating that this user account either:\n\n1. Has no roles explicitly granted to it\n2. Only has direct database permissions (not role-based access)\n\nIf you need to see what direct database permissions jsmith has instead, I can check that using a different query. Would you like me to look up the direct database permissions for jsmith?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_rolePermissions_missing_role", + "case_type": "missing_parameter", + "description": "Agent must ask which role to inspect", + "input": "What access rights does the role have?", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you check the access rights for a role, but I need to know which specific role you'd like me to look up.\n\nCould you please provide the name of the Teradata role you want to investigate?", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_userRoles_missing_user", + "case_type": "missing_parameter", + "description": "Agent must ask which user to retrieve roles for", + "input": "Show me the user roles", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'll retrieve the list of user roles from the system. However, I need to clarify: would you like to see:\n\n1. **All roles that exist in the system** (role definitions)\n2. **Roles assigned to a specific user** (requires a username)\n3. **Permissions granted to a specific role** (requires a role name)\n\nCould you please clarify which information you're looking for? If you want to see roles for a specific user or details about a specific role, please provide the username or role name.", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + }, + { + "case_id": "sec_userRoles_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which user, then calls sec_userRoles after user clarifies", + "input": "Turn 1: Show me the user roles | Turn 2: Which roles are assigned to user diana?", + "expected_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "diana" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "diana" + } + } + ], + "actual_output": "The query returned no results, which means **user 'diana' currently has no roles assigned**.\n\nThis could mean:\n1. The user 'diana' exists but has no roles granted to them\n2. The user 'diana' doesn't exist in the system\n3. The user has direct database permissions instead of role-based permissions\n\nWould you like me to:\n- Check what **direct database permissions** user diana has (using `sec_userDbPermissions`)?\n- Verify if the user exists by checking session information?\n- Look for a different user name (in case of a typo)?", + "metric_reasons": [], + "recommendation": null, + "turn_details": [ + { + "turn": 1, + "input": "Show me the user roles", + "mode": "clarification", + "passed": true, + "actual_tools": [] + }, + { + "turn": 2, + "input": "Which roles are assigned to user diana?", + "mode": "tool", + "passed": true, + "actual_tools": [ + { + "name": "sec_userRoles", + "params": { + "user_name": "diana" + } + } + ] + } + ] + }, + { + "case_id": "sec_rolePermissions_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which role, then calls sec_rolePermissions after user clarifies", + "input": "Turn 1: What access rights does the role have? | Turn 2: What database access has the analyst role been granted?", + "expected_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ], + "actual_output": "The query completed successfully, but **no database access permissions were found for the \"analyst\" role**. This could mean:\n\n1. **The role doesn't exist** - There may not be a role named \"analyst\" in the system\n2. **The role has no permissions** - The role exists but hasn't been granted any database access rights yet\n3. **Case sensitivity** - The role name might have different casing (e.g., \"Analyst\", \"ANALYST\")\n\nWould you like me to:\n- Check if the role exists with a different case variation?\n- List all available roles in the system?\n- Check permissions for a different role name?\n\nPlease let me know how you'd like to proceed!", + "metric_reasons": [], + "recommendation": null, + "turn_details": [ + { + "turn": 1, + "input": "What access rights does the role have?", + "mode": "clarification", + "passed": true, + "actual_tools": [] + }, + { + "turn": 2, + "input": "What database access has the analyst role been granted?", + "mode": "tool", + "passed": true, + "actual_tools": [ + { + "name": "sec_rolePermissions", + "params": { + "role_name": "analyst" + } + } + ] + } + ] + }, + { + "case_id": "sec_userDbPermissions_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which user, then calls sec_userDbPermissions after user clarifies", + "input": "Turn 1: Show me the database permissions | Turn 2: What databases can user bob access and what can they do in each?", + "expected_tools": [ + { + "name": "sec_userDbPermissions", + "params": { + "user_name": "bob" + } + } + ], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [ + { + "name": "sec_userDbPermissions", + "params": { + "user_name": "bob" + } + } + ], + "actual_output": "The query results show that user **'bob'** has **no direct database permissions** granted in the system. This means:\n\n- No direct SELECT, INSERT, UPDATE, DELETE, or other access rights are assigned to 'bob' at the database or table level\n- Bob may still have access through **role-based permissions** (roles assigned to the user that have permissions)\n\n**Would you like me to check:**\n1. **What roles are assigned to bob?** (using `sec_userRoles`) - This would show if bob has any roles that might grant permissions\n2. **The permissions of specific roles** that bob might belong to?\n\nThis would give us a complete picture of bob's effective access rights in the system.", + "metric_reasons": [], + "recommendation": null, + "turn_details": [ + { + "turn": 1, + "input": "Show me the database permissions", + "mode": "clarification", + "passed": true, + "actual_tools": [] + }, + { + "turn": 2, + "input": "What databases can user bob access and what can they do in each?", + "mode": "tool", + "passed": true, + "actual_tools": [ + { + "name": "sec_userDbPermissions", + "params": { + "user_name": "bob" + } + } + ] + } + ] + }, + { + "case_id": "sec_unknown_tool_missing", + "case_type": "missing_parameter", + "description": "Vague access question with no user or role named \u2014 agent should ask for clarification", + "input": "What access does this user have in the system?", + "expected_tools": [], + "passed": true, + "failure_stage": null, + "failure_detail": null, + "actual_tools": [], + "actual_output": "I'd be happy to help you check what access a user has in the system. However, I need to know which user you'd like me to look up.\n\nCould you please provide the username you want to check?\n\nOnce you give me the username, I can show you:\n- **Database permissions** - What access rights (SELECT, INSERT, UPDATE, DELETE, etc.) the user has across databases\n- **Role assignments** - Which roles are assigned to the user\n- **Role permissions** - What access those roles provide", + "metric_reasons": [], + "recommendation": null, + "turn_details": null + } + ] +} diff --git a/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.md b/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.md new file mode 100644 index 0000000..d0fc217 --- /dev/null +++ b/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.md @@ -0,0 +1,268 @@ +# Teradata MCP Eval Run Summary + +**Started (UTC):** 2026-06-23T21:14:19.042368+00:00 +**Module filter:** all +**Case type filter:** all +**Agent model:** us.anthropic.claude-sonnet-4-5-20250929-v1:0 +**Judge model:** us.anthropic.claude-sonnet-4-5-20250929-v1:0 +**Eval database:** demo_user +**Tool descriptions:** live MCP server (baseline) + +## Overview + +| Metric | Count | +| --- | ---: | +| Total cases | 106 | +| Passed | 103 | +| Failed | 3 | + +## Failed cases + +### base_tableList_missing_database (missing_parameter) + +**Description:** User asks for tables without specifying a database + +**Eval prompt:** + +> Show me what tables are available + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +- `base_tableList` with params: +```json +{ + "database_name": "" +} +``` + +**Agent response (excerpt):** + +> I can see the system has many tables available. The results show tables across multiple databases. Here's a summary of what I found: + +**Main Databases with Tables:** +- **DBC** - System database with metadata tables (InDoubtLog, StatsTbl, QryLog, DBCInfo, etc.) +- **TD_SYSFNLIB** - System function library +- **SYSLIB** - System library +- **TD_SERVER_DB** - Server database +- **SQLJ** - SQL/Java support +- **TD_SYSGPL** - System GPL (General Purpose Library) +- **SystemFe** - System functions +- **SYSSP + +**Failure (deterministic):** expected no tool calls for missing_parameter case, got ['base_tableList'] + +**Recommendation:** + +The prompt deliberately omits required information: "Show me what tables are available". The agent called `base_tableList` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response. + +### base_tableList_clarify_then_call (missing_parameter) + +**Description:** Agent asks which database, then calls base_tableList after user clarifies + +**Eval prompt:** + +> Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +- `base_tableList` with params: +```json +{ + "database_name": "" +} +``` + +**Agent response (excerpt):** + +> I can see there are tables available in the system. The results show many tables from the **DBC** database (which is the Teradata Data Dictionary/system catalog). + +Would you like me to: +1. **Filter by a specific database** - If you're looking for user/application tables rather than system tables, please tell me which database to show +2. **Show only user databases** - I can list the available databases first so you can pick one +3. **Continue showing all tables** - though the full list is quite l + +**Failure (deterministic):** turn 1: expected no tool calls for missing_parameter case, got ['base_tableList'] + +**Turn details:** + +```json +[ + { + "turn": 1, + "input": "Show me what tables are available", + "mode": "clarification", + "passed": false, + "failure_stage": "deterministic", + "failure_detail": "expected no tool calls for missing_parameter case, got ['base_tableList']", + "actual_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "" + } + } + ] + } +] +``` + +**Recommendation:** + +The prompt deliberately omits required information: "(multi-turn — see turn details)". The agent called `base_tableList` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response. + +### qlty_distinctCategories_missing_column (missing_parameter) + +**Description:** Agent must ask which column when none is specified + +**Eval prompt:** + +> What distinct values exist in a column of demo_user.evals_orders? + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +- `base_columnDescription` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Agent response (excerpt):** + +> The `demo_user.evals_orders` table has the following columns: + +1. **product_category** (VARCHAR) +2. **ship_date** (DATE) +3. **customer_name** (VARCHAR) +4. **order_id** (INTEGER) +5. **quantity** (INTEGER) +6. **amount** (DECIMAL) +7. **order_date** (DATE) + +Which column would you like to see the distinct values for? + +**Failure (deterministic):** expected no tool calls for missing_parameter case, got ['base_columnDescription'] + +**Recommendation:** + +The prompt deliberately omits required information: "What distinct values exist in a column of demo_user.evals_orders?". The agent called `base_columnDescription` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response. + +## Passed cases + +- `base_readQuery_happy` (happy_path) +- `base_readQuery_ambiguous_vs_tablePreview` (ambiguous_selection) +- `base_readQuery_missing_sql` (missing_parameter) +- `base_tableList_happy` (happy_path) +- `base_tableDDL_happy` (happy_path) +- `base_tableDDL_ambiguous_vs_columnMetadata` (ambiguous_selection) +- `base_columnMetadata_happy` (happy_path) +- `base_columnMetadata_ambiguous_vs_tableDDL` (ambiguous_selection) +- `base_tablePreview_happy` (happy_path) +- `base_tablePreview_ambiguous_vs_readQuery` (ambiguous_selection) +- `base_databaseList_happy` (happy_path) +- `base_tableList_ambiguous_vs_databaseList` (ambiguous_selection) +- `base_databaseList_ambiguous_vs_tableList` (ambiguous_selection) +- `base_saveDDL_ambiguous_vs_tableDDL` (ambiguous_selection) +- `base_tableAffinity_ambiguous_vs_tableUsage` (ambiguous_selection) +- `base_tableUsage_ambiguous_vs_tableAffinity` (ambiguous_selection) +- `base_saveDDL_happy` (happy_path) +- `base_tableAffinity_happy` (happy_path) +- `base_tableUsage_happy` (happy_path) +- `base_multi_tool_list_then_preview` (multi_tool) +- `base_multi_tool_ddl_then_columns` (multi_tool) +- `base_tableDDL_missing_table` (missing_parameter) +- `base_tablePreview_missing_table` (missing_parameter) +- `base_saveDDL_missing_table` (missing_parameter) +- `base_tablePreview_clarify_then_call` (missing_parameter) +- `base_readQuery_clarify_then_call` (missing_parameter) +- `base_columnDescription_happy` (happy_path) +- `base_columnDescription_ambiguous_vs_columnMetadata` (ambiguous_selection) +- `base_columnMetadata_ambiguous_vs_columnDescription` (ambiguous_selection) +- `base_unknown_tool_missing` (missing_parameter) +- `chat_completeChat_missing_message` (missing_parameter) +- `chat_unknown_tool_missing` (missing_parameter) +- `dba_tableSpace_happy` (happy_path) +- `dba_tableSqlList_happy` (happy_path) +- `dba_userSqlList_happy` (happy_path) +- `dba_databaseSpace_happy` (happy_path) +- `dba_tableUsageImpact_happy` (happy_path) +- `dba_resusageSummary_happy` (happy_path) +- `dba_databaseVersion_happy` (happy_path) +- `dba_flowControl_happy` (happy_path) +- `dba_featureUsage_happy` (happy_path) +- `dba_userDelay_happy` (happy_path) +- `dba_sessionInfo_happy` (happy_path) +- `dba_systemSpace_happy` (happy_path) +- `dba_tableSpace_ambiguous_vs_databaseSpace` (ambiguous_selection) +- `dba_databaseSpace_ambiguous_vs_systemSpace` (ambiguous_selection) +- `dba_systemSpace_ambiguous_vs_databaseSpace` (ambiguous_selection) +- `dba_tableSqlList_ambiguous_vs_userSqlList` (ambiguous_selection) +- `dba_userSqlList_ambiguous_vs_tableSqlList` (ambiguous_selection) +- `dba_tableUsageImpact_ambiguous_vs_resusageSummary` (ambiguous_selection) +- `dba_resusageSummary_ambiguous_vs_tableUsageImpact` (ambiguous_selection) +- `dba_tableUsageImpact_ambiguous_vs_sessionInfo` (ambiguous_selection) +- `dba_sessionInfo_ambiguous_vs_tableUsageImpact` (ambiguous_selection) +- `dba_userDelay_ambiguous_vs_flowControl` (ambiguous_selection) +- `dba_flowControl_ambiguous_vs_userDelay` (ambiguous_selection) +- `dba_tableSpace_missing_db` (missing_parameter) +- `dba_tableSqlList_missing_table` (missing_parameter) +- `dba_userSqlList_missing_user` (missing_parameter) +- `dba_databaseSpace_missing_db` (missing_parameter) +- `dba_tableUsageImpact_missing_db` (missing_parameter) +- `dba_databaseSpace_clarify_then_call` (missing_parameter) +- `dba_tableSqlList_clarify_then_call` (missing_parameter) +- `dba_userSqlList_clarify_then_call` (missing_parameter) +- `dba_unknown_tool_missing` (missing_parameter) +- `plot_line_chart_happy` (happy_path) +- `plot_pie_chart_happy` (happy_path) +- `plot_polar_chart_happy` (happy_path) +- `plot_radar_chart_happy` (happy_path) +- `plot_line_vs_radar_ambiguous` (ambiguous_selection) +- `plot_pie_vs_polar_ambiguous` (ambiguous_selection) +- `plot_missing_data` (missing_parameter) +- `plot_pie_chart_direct` (happy_path) +- `plot_missing_table` (missing_parameter) +- `plot_unknown_tool_missing` (missing_parameter) +- `qlty_missingValues_happy` (happy_path) +- `qlty_missingValues_ambiguous_vs_rowsWithMissingValues` (ambiguous_selection) +- `qlty_negativeValues_happy` (happy_path) +- `qlty_distinctCategories_happy` (happy_path) +- `qlty_standardDeviation_happy` (happy_path) +- `qlty_standardDeviation_ambiguous_vs_univariateStatistics` (ambiguous_selection) +- `qlty_columnSummary_happy` (happy_path) +- `qlty_univariateStatistics_happy` (happy_path) +- `qlty_univariateStatistics_ambiguous_vs_columnSummary` (ambiguous_selection) +- `qlty_rowsWithMissingValues_happy` (happy_path) +- `qlty_rowsWithMissingValues_ambiguous_vs_missingValues` (ambiguous_selection) +- `qlty_multi_tool_summary_then_nulls` (multi_tool) +- `qlty_unknown_tool_missing` (missing_parameter) +- `sec_userDbPermissions_ambiguous_vs_userRoles` (ambiguous_selection) +- `sec_userDbPermissions_missing_user` (missing_parameter) +- `sec_rolePermissions_ambiguous_vs_userDbPermissions` (ambiguous_selection) +- `sec_userRoles_ambiguous_vs_rolePermissions` (ambiguous_selection) +- `sec_rolePermissions_ambiguous_vs_userRoles` (ambiguous_selection) +- `sec_userRoles_ambiguous_vs_userDbPermissions` (ambiguous_selection) +- `sec_multi_tool_user_roles_then_role_permissions` (multi_tool) +- `sec_userDbPermissions_happy` (happy_path) +- `sec_rolePermissions_happy` (happy_path) +- `sec_userRoles_happy` (happy_path) +- `sec_rolePermissions_missing_role` (missing_parameter) +- `sec_userRoles_missing_user` (missing_parameter) +- `sec_userRoles_clarify_then_call` (missing_parameter) +- `sec_rolePermissions_clarify_then_call` (missing_parameter) +- `sec_userDbPermissions_clarify_then_call` (missing_parameter) +- `sec_unknown_tool_missing` (missing_parameter) + diff --git a/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/manifest.json b/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/manifest.json new file mode 100644 index 0000000..7f7bf6a --- /dev/null +++ b/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/manifest.json @@ -0,0 +1,22 @@ +{ + "run_id": "2026-06-24T00-05-13.720459Z__base__baseline", + "started_at": "2026-06-24T00:05:13.720459+00:00", + "module_filter": "base", + "case_type_filter": "all", + "run_label": null, + "description_mode": "mcp_server", + "description_overrides_file": null, + "description_override_count": 0, + "agent_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "judge_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "evals_database": "demo_user", + "total": 32, + "passed": 0, + "failed": 32, + "artifacts": { + "summary_md": "summary.md", + "summary_json": "summary.json", + "manifest": "manifest.json", + "suggested_overrides": "suggested_overrides.json" + } +} diff --git a/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.json b/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.json new file mode 100644 index 0000000..f537d98 --- /dev/null +++ b/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.json @@ -0,0 +1,683 @@ +{ + "run_id": "2026-06-24T00-05-13.720459Z__base__baseline", + "started_at": "2026-06-24T00:05:13.720459+00:00", + "module_filter": "base", + "case_type_filter": "all", + "run_label": null, + "agent_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "judge_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "evals_database": "demo_user", + "description_mode": "mcp_server", + "description_overrides_file": null, + "description_override_count": 0, + "total": 32, + "passed": 0, + "failed": 32, + "cases": [ + { + "case_id": "base_readQuery_happy", + "case_type": "happy_path", + "description": "Execute a SQL query against the evals employees table", + "input": "Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_readQuery_ambiguous_vs_tablePreview", + "case_type": "ambiguous_selection", + "description": "Prompt with a WHERE clause should use base_readQuery, not base_tablePreview", + "input": "Get me all orders from demo_user.evals_orders where the amount is greater than 500", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_readQuery_missing_sql", + "case_type": "missing_parameter", + "description": "Agent must ask for clarification when no SQL or table is given", + "input": "Query the database for me", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_happy", + "case_type": "happy_path", + "description": "List tables in the evals database", + "input": "What tables are in the demo_user database?", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_missing_database", + "case_type": "missing_parameter", + "description": "User asks for tables without specifying a database", + "input": "Show me what tables are available", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableDDL_happy", + "case_type": "happy_path", + "description": "Retrieve DDL for the evals employees table", + "input": "Show me the CREATE statement for evals_employees in the demo_user database", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableDDL_ambiguous_vs_columnMetadata", + "case_type": "ambiguous_selection", + "description": "Asking for full structure including constraints should prefer base_tableDDL", + "input": "Describe the full structure of demo_user.evals_employees including any constraints and indexes", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnMetadata_happy", + "case_type": "happy_path", + "description": "Get precise Teradata type codes, character sets, and precision for columns in evals_orders", + "input": "Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnMetadata_ambiguous_vs_tableDDL", + "case_type": "ambiguous_selection", + "description": "Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL", + "input": "I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_happy", + "case_type": "happy_path", + "description": "Preview a sample of rows from the evals orders table", + "input": "Give me a quick preview of what's in demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_ambiguous_vs_readQuery", + "case_type": "ambiguous_selection", + "description": "A simple 'show me a few rows' without a WHERE clause should use base_tablePreview", + "input": "Show me a few rows from demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_databaseList_happy", + "case_type": "happy_path", + "description": "List all accessible databases", + "input": "What databases do I have access to?", + "expected_tools": [ + { + "name": "base_databaseList", + "params": {} + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_ambiguous_vs_databaseList", + "case_type": "ambiguous_selection", + "description": "Listing tables inside one database should prefer base_tableList, not base_databaseList", + "input": "What tables exist inside the demo_user database?", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_databaseList_ambiguous_vs_tableList", + "case_type": "ambiguous_selection", + "description": "Listing accessible databases should prefer base_databaseList, not base_tableList", + "input": "Which databases am I allowed to connect to on this system?", + "expected_tools": [ + { + "name": "base_databaseList", + "params": {} + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_saveDDL_ambiguous_vs_tableDDL", + "case_type": "ambiguous_selection", + "description": "Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL", + "input": "Export the DDL for demo_user.evals_orders to a file on disk", + "expected_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableAffinity_ambiguous_vs_tableUsage", + "case_type": "ambiguous_selection", + "description": "Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage", + "input": "Which tables in demo_user tend to appear in the same queries as evals_employees?", + "expected_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableUsage_ambiguous_vs_tableAffinity", + "case_type": "ambiguous_selection", + "description": "Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity", + "input": "Who has been reading tables in demo_user lately and how often?", + "expected_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_saveDDL_happy", + "case_type": "happy_path", + "description": "Save DDL for the evals employees table to a file", + "input": "Save the DDL for evals_employees in demo_user to a file", + "expected_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableAffinity_happy", + "case_type": "happy_path", + "description": "Find tables commonly queried together with evals_orders", + "input": "Which tables in demo_user are most often queried together with evals_orders?", + "expected_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableUsage_happy", + "case_type": "happy_path", + "description": "Check usage metrics for the evals database", + "input": "How frequently are the tables in demo_user being accessed and by whom?", + "expected_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_multi_tool_list_then_preview", + "case_type": "multi_tool", + "description": "List tables in evals database then preview one", + "input": "List all tables in demo_user, then show me a preview of evals_employees", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + }, + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_multi_tool_ddl_then_columns", + "case_type": "multi_tool", + "description": "Get DDL then drill into column detail for the same table", + "input": "First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + }, + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableDDL_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask for database name when only table is provided", + "input": "Show me the CREATE statement for evals_employees", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask which table to preview", + "input": "Preview some rows", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_saveDDL_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask which table to save DDL for", + "input": "Save the DDL to a file", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which table to preview, then calls base_tablePreview after user clarifies", + "input": "Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which database, then calls base_tableList after user clarifies", + "input": "Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_readQuery_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks for SQL, then calls base_readQuery after user provides the query", + "input": "Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnDescription_happy", + "case_type": "happy_path", + "description": "Show basic column info for the evals employees table", + "input": "Show me the columns in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnDescription_ambiguous_vs_columnMetadata", + "case_type": "ambiguous_selection", + "description": "A simple 'what columns does this table have' question should prefer base_columnDescription", + "input": "What are the column names and types in demo_user.evals_orders?", + "expected_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnMetadata_ambiguous_vs_columnDescription", + "case_type": "ambiguous_selection", + "description": "Asking for precise Teradata type codes and character sets should prefer base_columnMetadata", + "input": "Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_unknown_tool_missing", + "case_type": "missing_parameter", + "description": "User asks to do something no tool supports \u2014 agent should explain and ask for clarification", + "input": "Automatically tune my Teradata database indexes and rebalance the AMPs", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + } + ] +} diff --git a/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.md b/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.md new file mode 100644 index 0000000..e153c05 --- /dev/null +++ b/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.md @@ -0,0 +1,867 @@ +# Teradata MCP Eval Run Summary + +**Started (UTC):** 2026-06-24T00:05:13.720459+00:00 +**Module filter:** base +**Case type filter:** all +**Agent model:** anthropic.claude-3-5-sonnet-20241022-v2:0 +**Judge model:** anthropic.claude-3-5-sonnet-20241022-v2:0 +**Eval database:** demo_user +**Tool descriptions:** live MCP server (baseline) + +## Overview + +| Metric | Count | +| --- | ---: | +| Total cases | 32 | +| Passed | 0 | +| Failed | 32 | + +## Failed cases + +### base_readQuery_happy (happy_path) + +**Description:** Execute a SQL query against the evals employees table + +**Eval prompt:** + +> Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales' + +**Expected tool(s):** + +- `base_readQuery` with params: +```json +{ + "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_readQuery_ambiguous_vs_tablePreview (ambiguous_selection) + +**Description:** Prompt with a WHERE clause should use base_readQuery, not base_tablePreview + +**Eval prompt:** + +> Get me all orders from demo_user.evals_orders where the amount is greater than 500 + +**Expected tool(s):** + +- `base_readQuery` with params: +```json +{ + "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_readQuery_missing_sql (missing_parameter) + +**Description:** Agent must ask for clarification when no SQL or table is given + +**Eval prompt:** + +> Query the database for me + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_happy (happy_path) + +**Description:** List tables in the evals database + +**Eval prompt:** + +> What tables are in the demo_user database? + +**Expected tool(s):** + +- `base_tableList` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_missing_database (missing_parameter) + +**Description:** User asks for tables without specifying a database + +**Eval prompt:** + +> Show me what tables are available + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableDDL_happy (happy_path) + +**Description:** Retrieve DDL for the evals employees table + +**Eval prompt:** + +> Show me the CREATE statement for evals_employees in the demo_user database + +**Expected tool(s):** + +- `base_tableDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableDDL_ambiguous_vs_columnMetadata (ambiguous_selection) + +**Description:** Asking for full structure including constraints should prefer base_tableDDL + +**Eval prompt:** + +> Describe the full structure of demo_user.evals_employees including any constraints and indexes + +**Expected tool(s):** + +- `base_tableDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnMetadata_happy (happy_path) + +**Description:** Get precise Teradata type codes, character sets, and precision for columns in evals_orders + +**Eval prompt:** + +> Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders + +**Expected tool(s):** + +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnMetadata_ambiguous_vs_tableDDL (ambiguous_selection) + +**Description:** Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL + +**Eval prompt:** + +> I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees + +**Expected tool(s):** + +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_happy (happy_path) + +**Description:** Preview a sample of rows from the evals orders table + +**Eval prompt:** + +> Give me a quick preview of what's in demo_user.evals_orders + +**Expected tool(s):** + +- `base_tablePreview` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_ambiguous_vs_readQuery (ambiguous_selection) + +**Description:** A simple 'show me a few rows' without a WHERE clause should use base_tablePreview + +**Eval prompt:** + +> Show me a few rows from demo_user.evals_orders + +**Expected tool(s):** + +- `base_tablePreview` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_databaseList_happy (happy_path) + +**Description:** List all accessible databases + +**Eval prompt:** + +> What databases do I have access to? + +**Expected tool(s):** + +- `base_databaseList` with params: +```json +{} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_ambiguous_vs_databaseList (ambiguous_selection) + +**Description:** Listing tables inside one database should prefer base_tableList, not base_databaseList + +**Eval prompt:** + +> What tables exist inside the demo_user database? + +**Expected tool(s):** + +- `base_tableList` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_databaseList_ambiguous_vs_tableList (ambiguous_selection) + +**Description:** Listing accessible databases should prefer base_databaseList, not base_tableList + +**Eval prompt:** + +> Which databases am I allowed to connect to on this system? + +**Expected tool(s):** + +- `base_databaseList` with params: +```json +{} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_saveDDL_ambiguous_vs_tableDDL (ambiguous_selection) + +**Description:** Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL + +**Eval prompt:** + +> Export the DDL for demo_user.evals_orders to a file on disk + +**Expected tool(s):** + +- `base_saveDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableAffinity_ambiguous_vs_tableUsage (ambiguous_selection) + +**Description:** Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage + +**Eval prompt:** + +> Which tables in demo_user tend to appear in the same queries as evals_employees? + +**Expected tool(s):** + +- `base_tableAffinity` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableUsage_ambiguous_vs_tableAffinity (ambiguous_selection) + +**Description:** Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity + +**Eval prompt:** + +> Who has been reading tables in demo_user lately and how often? + +**Expected tool(s):** + +- `base_tableUsage` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_saveDDL_happy (happy_path) + +**Description:** Save DDL for the evals employees table to a file + +**Eval prompt:** + +> Save the DDL for evals_employees in demo_user to a file + +**Expected tool(s):** + +- `base_saveDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableAffinity_happy (happy_path) + +**Description:** Find tables commonly queried together with evals_orders + +**Eval prompt:** + +> Which tables in demo_user are most often queried together with evals_orders? + +**Expected tool(s):** + +- `base_tableAffinity` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableUsage_happy (happy_path) + +**Description:** Check usage metrics for the evals database + +**Eval prompt:** + +> How frequently are the tables in demo_user being accessed and by whom? + +**Expected tool(s):** + +- `base_tableUsage` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_multi_tool_list_then_preview (multi_tool) + +**Description:** List tables in evals database then preview one + +**Eval prompt:** + +> List all tables in demo_user, then show me a preview of evals_employees + +**Expected tool(s):** + +- `base_tableList` with params: +```json +{ + "database_name": "demo_user" +} +``` +- `base_tablePreview` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_multi_tool_ddl_then_columns (multi_tool) + +**Description:** Get DDL then drill into column detail for the same table + +**Eval prompt:** + +> First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata + +**Expected tool(s):** + +- `base_tableDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableDDL_missing_table (missing_parameter) + +**Description:** Agent must ask for database name when only table is provided + +**Eval prompt:** + +> Show me the CREATE statement for evals_employees + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_missing_table (missing_parameter) + +**Description:** Agent must ask which table to preview + +**Eval prompt:** + +> Preview some rows + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_saveDDL_missing_table (missing_parameter) + +**Description:** Agent must ask which table to save DDL for + +**Eval prompt:** + +> Save the DDL to a file + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_clarify_then_call (missing_parameter) + +**Description:** Agent asks which table to preview, then calls base_tablePreview after user clarifies + +**Eval prompt:** + +> Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_clarify_then_call (missing_parameter) + +**Description:** Agent asks which database, then calls base_tableList after user clarifies + +**Eval prompt:** + +> Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_readQuery_clarify_then_call (missing_parameter) + +**Description:** Agent asks for SQL, then calls base_readQuery after user provides the query + +**Eval prompt:** + +> Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales' + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnDescription_happy (happy_path) + +**Description:** Show basic column info for the evals employees table + +**Eval prompt:** + +> Show me the columns in demo_user.evals_employees + +**Expected tool(s):** + +- `base_columnDescription` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnDescription_ambiguous_vs_columnMetadata (ambiguous_selection) + +**Description:** A simple 'what columns does this table have' question should prefer base_columnDescription + +**Eval prompt:** + +> What are the column names and types in demo_user.evals_orders? + +**Expected tool(s):** + +- `base_columnDescription` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnMetadata_ambiguous_vs_columnDescription (ambiguous_selection) + +**Description:** Asking for precise Teradata type codes and character sets should prefer base_columnMetadata + +**Eval prompt:** + +> Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees + +**Expected tool(s):** + +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_unknown_tool_missing (missing_parameter) + +**Description:** User asks to do something no tool supports — agent should explain and ask for clarification + +**Eval prompt:** + +> Automatically tune my Teradata database indexes and rebalance the AMPs + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +## Passed cases + +_None._ + diff --git a/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/manifest.json b/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/manifest.json new file mode 100644 index 0000000..6626fe8 --- /dev/null +++ b/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/manifest.json @@ -0,0 +1,22 @@ +{ + "run_id": "2026-06-24T00-06-59.222908Z__base__baseline", + "started_at": "2026-06-24T00:06:59.222908+00:00", + "module_filter": "base", + "case_type_filter": "all", + "run_label": null, + "description_mode": "mcp_server", + "description_overrides_file": null, + "description_override_count": 0, + "agent_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "judge_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "evals_database": "demo_user", + "total": 32, + "passed": 0, + "failed": 32, + "artifacts": { + "summary_md": "summary.md", + "summary_json": "summary.json", + "manifest": "manifest.json", + "suggested_overrides": "suggested_overrides.json" + } +} diff --git a/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json b/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json new file mode 100644 index 0000000..90bd6d6 --- /dev/null +++ b/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json @@ -0,0 +1,683 @@ +{ + "run_id": "2026-06-24T00-06-59.222908Z__base__baseline", + "started_at": "2026-06-24T00:06:59.222908+00:00", + "module_filter": "base", + "case_type_filter": "all", + "run_label": null, + "agent_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "judge_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "evals_database": "demo_user", + "description_mode": "mcp_server", + "description_overrides_file": null, + "description_override_count": 0, + "total": 32, + "passed": 0, + "failed": 32, + "cases": [ + { + "case_id": "base_readQuery_happy", + "case_type": "happy_path", + "description": "Execute a SQL query against the evals employees table", + "input": "Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_readQuery_ambiguous_vs_tablePreview", + "case_type": "ambiguous_selection", + "description": "Prompt with a WHERE clause should use base_readQuery, not base_tablePreview", + "input": "Get me all orders from demo_user.evals_orders where the amount is greater than 500", + "expected_tools": [ + { + "name": "base_readQuery", + "params": { + "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_readQuery_missing_sql", + "case_type": "missing_parameter", + "description": "Agent must ask for clarification when no SQL or table is given", + "input": "Query the database for me", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_happy", + "case_type": "happy_path", + "description": "List tables in the evals database", + "input": "What tables are in the demo_user database?", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_missing_database", + "case_type": "missing_parameter", + "description": "User asks for tables without specifying a database", + "input": "Show me what tables are available", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableDDL_happy", + "case_type": "happy_path", + "description": "Retrieve DDL for the evals employees table", + "input": "Show me the CREATE statement for evals_employees in the demo_user database", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableDDL_ambiguous_vs_columnMetadata", + "case_type": "ambiguous_selection", + "description": "Asking for full structure including constraints should prefer base_tableDDL", + "input": "Describe the full structure of demo_user.evals_employees including any constraints and indexes", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnMetadata_happy", + "case_type": "happy_path", + "description": "Get precise Teradata type codes, character sets, and precision for columns in evals_orders", + "input": "Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnMetadata_ambiguous_vs_tableDDL", + "case_type": "ambiguous_selection", + "description": "Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL", + "input": "I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_happy", + "case_type": "happy_path", + "description": "Preview a sample of rows from the evals orders table", + "input": "Give me a quick preview of what's in demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_ambiguous_vs_readQuery", + "case_type": "ambiguous_selection", + "description": "A simple 'show me a few rows' without a WHERE clause should use base_tablePreview", + "input": "Show me a few rows from demo_user.evals_orders", + "expected_tools": [ + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_databaseList_happy", + "case_type": "happy_path", + "description": "List all accessible databases", + "input": "What databases do I have access to?", + "expected_tools": [ + { + "name": "base_databaseList", + "params": {} + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_ambiguous_vs_databaseList", + "case_type": "ambiguous_selection", + "description": "Listing tables inside one database should prefer base_tableList, not base_databaseList", + "input": "What tables exist inside the demo_user database?", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_databaseList_ambiguous_vs_tableList", + "case_type": "ambiguous_selection", + "description": "Listing accessible databases should prefer base_databaseList, not base_tableList", + "input": "Which databases am I allowed to connect to on this system?", + "expected_tools": [ + { + "name": "base_databaseList", + "params": {} + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_saveDDL_ambiguous_vs_tableDDL", + "case_type": "ambiguous_selection", + "description": "Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL", + "input": "Export the DDL for demo_user.evals_orders to a file on disk", + "expected_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableAffinity_ambiguous_vs_tableUsage", + "case_type": "ambiguous_selection", + "description": "Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage", + "input": "Which tables in demo_user tend to appear in the same queries as evals_employees?", + "expected_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableUsage_ambiguous_vs_tableAffinity", + "case_type": "ambiguous_selection", + "description": "Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity", + "input": "Who has been reading tables in demo_user lately and how often?", + "expected_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_saveDDL_happy", + "case_type": "happy_path", + "description": "Save DDL for the evals employees table to a file", + "input": "Save the DDL for evals_employees in demo_user to a file", + "expected_tools": [ + { + "name": "base_saveDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableAffinity_happy", + "case_type": "happy_path", + "description": "Find tables commonly queried together with evals_orders", + "input": "Which tables in demo_user are most often queried together with evals_orders?", + "expected_tools": [ + { + "name": "base_tableAffinity", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableUsage_happy", + "case_type": "happy_path", + "description": "Check usage metrics for the evals database", + "input": "How frequently are the tables in demo_user being accessed and by whom?", + "expected_tools": [ + { + "name": "base_tableUsage", + "params": { + "database_name": "demo_user" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_multi_tool_list_then_preview", + "case_type": "multi_tool", + "description": "List tables in evals database then preview one", + "input": "List all tables in demo_user, then show me a preview of evals_employees", + "expected_tools": [ + { + "name": "base_tableList", + "params": { + "database_name": "demo_user" + } + }, + { + "name": "base_tablePreview", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_multi_tool_ddl_then_columns", + "case_type": "multi_tool", + "description": "Get DDL then drill into column detail for the same table", + "input": "First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata", + "expected_tools": [ + { + "name": "base_tableDDL", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + }, + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableDDL_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask for database name when only table is provided", + "input": "Show me the CREATE statement for evals_employees", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask which table to preview", + "input": "Preview some rows", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_saveDDL_missing_table", + "case_type": "missing_parameter", + "description": "Agent must ask which table to save DDL for", + "input": "Save the DDL to a file", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tablePreview_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which table to preview, then calls base_tablePreview after user clarifies", + "input": "Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_tableList_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks which database, then calls base_tableList after user clarifies", + "input": "Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_readQuery_clarify_then_call", + "case_type": "missing_parameter", + "description": "Agent asks for SQL, then calls base_readQuery after user provides the query", + "input": "Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnDescription_happy", + "case_type": "happy_path", + "description": "Show basic column info for the evals employees table", + "input": "Show me the columns in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnDescription_ambiguous_vs_columnMetadata", + "case_type": "ambiguous_selection", + "description": "A simple 'what columns does this table have' question should prefer base_columnDescription", + "input": "What are the column names and types in demo_user.evals_orders?", + "expected_tools": [ + { + "name": "base_columnDescription", + "params": { + "database_name": "demo_user", + "table_name": "evals_orders" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_columnMetadata_ambiguous_vs_columnDescription", + "case_type": "ambiguous_selection", + "description": "Asking for precise Teradata type codes and character sets should prefer base_columnMetadata", + "input": "Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees", + "expected_tools": [ + { + "name": "base_columnMetadata", + "params": { + "database_name": "demo_user", + "object_name": "evals_employees" + } + } + ], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + }, + { + "case_id": "base_unknown_tool_missing", + "case_type": "missing_parameter", + "description": "User asks to do something no tool supports \u2014 agent should explain and ask for clarification", + "input": "Automatically tune my Teradata database indexes and rebalance the AMPs", + "expected_tools": [], + "passed": false, + "failure_stage": "agent", + "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", + "actual_tools": null, + "actual_output": null, + "metric_reasons": [], + "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", + "turn_details": null + } + ] +} diff --git a/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md b/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md new file mode 100644 index 0000000..9973a9d --- /dev/null +++ b/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md @@ -0,0 +1,867 @@ +# Teradata MCP Eval Run Summary + +**Started (UTC):** 2026-06-24T00:06:59.222908+00:00 +**Module filter:** base +**Case type filter:** all +**Agent model:** anthropic.claude-3-5-sonnet-20241022-v2:0 +**Judge model:** anthropic.claude-3-5-sonnet-20241022-v2:0 +**Eval database:** demo_user +**Tool descriptions:** live MCP server (baseline) + +## Overview + +| Metric | Count | +| --- | ---: | +| Total cases | 32 | +| Passed | 0 | +| Failed | 32 | + +## Failed cases + +### base_readQuery_happy (happy_path) + +**Description:** Execute a SQL query against the evals employees table + +**Eval prompt:** + +> Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales' + +**Expected tool(s):** + +- `base_readQuery` with params: +```json +{ + "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_readQuery_ambiguous_vs_tablePreview (ambiguous_selection) + +**Description:** Prompt with a WHERE clause should use base_readQuery, not base_tablePreview + +**Eval prompt:** + +> Get me all orders from demo_user.evals_orders where the amount is greater than 500 + +**Expected tool(s):** + +- `base_readQuery` with params: +```json +{ + "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_readQuery_missing_sql (missing_parameter) + +**Description:** Agent must ask for clarification when no SQL or table is given + +**Eval prompt:** + +> Query the database for me + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_happy (happy_path) + +**Description:** List tables in the evals database + +**Eval prompt:** + +> What tables are in the demo_user database? + +**Expected tool(s):** + +- `base_tableList` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_missing_database (missing_parameter) + +**Description:** User asks for tables without specifying a database + +**Eval prompt:** + +> Show me what tables are available + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableDDL_happy (happy_path) + +**Description:** Retrieve DDL for the evals employees table + +**Eval prompt:** + +> Show me the CREATE statement for evals_employees in the demo_user database + +**Expected tool(s):** + +- `base_tableDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableDDL_ambiguous_vs_columnMetadata (ambiguous_selection) + +**Description:** Asking for full structure including constraints should prefer base_tableDDL + +**Eval prompt:** + +> Describe the full structure of demo_user.evals_employees including any constraints and indexes + +**Expected tool(s):** + +- `base_tableDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnMetadata_happy (happy_path) + +**Description:** Get precise Teradata type codes, character sets, and precision for columns in evals_orders + +**Eval prompt:** + +> Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders + +**Expected tool(s):** + +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnMetadata_ambiguous_vs_tableDDL (ambiguous_selection) + +**Description:** Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL + +**Eval prompt:** + +> I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees + +**Expected tool(s):** + +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_happy (happy_path) + +**Description:** Preview a sample of rows from the evals orders table + +**Eval prompt:** + +> Give me a quick preview of what's in demo_user.evals_orders + +**Expected tool(s):** + +- `base_tablePreview` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_ambiguous_vs_readQuery (ambiguous_selection) + +**Description:** A simple 'show me a few rows' without a WHERE clause should use base_tablePreview + +**Eval prompt:** + +> Show me a few rows from demo_user.evals_orders + +**Expected tool(s):** + +- `base_tablePreview` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_databaseList_happy (happy_path) + +**Description:** List all accessible databases + +**Eval prompt:** + +> What databases do I have access to? + +**Expected tool(s):** + +- `base_databaseList` with params: +```json +{} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_ambiguous_vs_databaseList (ambiguous_selection) + +**Description:** Listing tables inside one database should prefer base_tableList, not base_databaseList + +**Eval prompt:** + +> What tables exist inside the demo_user database? + +**Expected tool(s):** + +- `base_tableList` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_databaseList_ambiguous_vs_tableList (ambiguous_selection) + +**Description:** Listing accessible databases should prefer base_databaseList, not base_tableList + +**Eval prompt:** + +> Which databases am I allowed to connect to on this system? + +**Expected tool(s):** + +- `base_databaseList` with params: +```json +{} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_saveDDL_ambiguous_vs_tableDDL (ambiguous_selection) + +**Description:** Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL + +**Eval prompt:** + +> Export the DDL for demo_user.evals_orders to a file on disk + +**Expected tool(s):** + +- `base_saveDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableAffinity_ambiguous_vs_tableUsage (ambiguous_selection) + +**Description:** Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage + +**Eval prompt:** + +> Which tables in demo_user tend to appear in the same queries as evals_employees? + +**Expected tool(s):** + +- `base_tableAffinity` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableUsage_ambiguous_vs_tableAffinity (ambiguous_selection) + +**Description:** Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity + +**Eval prompt:** + +> Who has been reading tables in demo_user lately and how often? + +**Expected tool(s):** + +- `base_tableUsage` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_saveDDL_happy (happy_path) + +**Description:** Save DDL for the evals employees table to a file + +**Eval prompt:** + +> Save the DDL for evals_employees in demo_user to a file + +**Expected tool(s):** + +- `base_saveDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableAffinity_happy (happy_path) + +**Description:** Find tables commonly queried together with evals_orders + +**Eval prompt:** + +> Which tables in demo_user are most often queried together with evals_orders? + +**Expected tool(s):** + +- `base_tableAffinity` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableUsage_happy (happy_path) + +**Description:** Check usage metrics for the evals database + +**Eval prompt:** + +> How frequently are the tables in demo_user being accessed and by whom? + +**Expected tool(s):** + +- `base_tableUsage` with params: +```json +{ + "database_name": "demo_user" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_multi_tool_list_then_preview (multi_tool) + +**Description:** List tables in evals database then preview one + +**Eval prompt:** + +> List all tables in demo_user, then show me a preview of evals_employees + +**Expected tool(s):** + +- `base_tableList` with params: +```json +{ + "database_name": "demo_user" +} +``` +- `base_tablePreview` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_multi_tool_ddl_then_columns (multi_tool) + +**Description:** Get DDL then drill into column detail for the same table + +**Eval prompt:** + +> First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata + +**Expected tool(s):** + +- `base_tableDDL` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableDDL_missing_table (missing_parameter) + +**Description:** Agent must ask for database name when only table is provided + +**Eval prompt:** + +> Show me the CREATE statement for evals_employees + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_missing_table (missing_parameter) + +**Description:** Agent must ask which table to preview + +**Eval prompt:** + +> Preview some rows + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_saveDDL_missing_table (missing_parameter) + +**Description:** Agent must ask which table to save DDL for + +**Eval prompt:** + +> Save the DDL to a file + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tablePreview_clarify_then_call (missing_parameter) + +**Description:** Agent asks which table to preview, then calls base_tablePreview after user clarifies + +**Eval prompt:** + +> Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_tableList_clarify_then_call (missing_parameter) + +**Description:** Agent asks which database, then calls base_tableList after user clarifies + +**Eval prompt:** + +> Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_readQuery_clarify_then_call (missing_parameter) + +**Description:** Agent asks for SQL, then calls base_readQuery after user provides the query + +**Eval prompt:** + +> Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales' + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnDescription_happy (happy_path) + +**Description:** Show basic column info for the evals employees table + +**Eval prompt:** + +> Show me the columns in demo_user.evals_employees + +**Expected tool(s):** + +- `base_columnDescription` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnDescription_ambiguous_vs_columnMetadata (ambiguous_selection) + +**Description:** A simple 'what columns does this table have' question should prefer base_columnDescription + +**Eval prompt:** + +> What are the column names and types in demo_user.evals_orders? + +**Expected tool(s):** + +- `base_columnDescription` with params: +```json +{ + "database_name": "demo_user", + "table_name": "evals_orders" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_columnMetadata_ambiguous_vs_columnDescription (ambiguous_selection) + +**Description:** Asking for precise Teradata type codes and character sets should prefer base_columnMetadata + +**Eval prompt:** + +> Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees + +**Expected tool(s):** + +- `base_columnMetadata` with params: +```json +{ + "database_name": "demo_user", + "object_name": "evals_employees" +} +``` + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +### base_unknown_tool_missing (missing_parameter) + +**Description:** User asks to do something no tool supports — agent should explain and ask for clarification + +**Eval prompt:** + +> Automatically tune my Teradata database indexes and rebalance the AMPs + +**Expected tool(s):** + +_none_ + +**Actual tool(s):** + +_none_ + +**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) + +**Recommendation:** + +The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). + +## Passed cases + +_None._ + diff --git a/tests/eval_test/run_evals.py b/tests/eval_test/run_evals.py new file mode 100644 index 0000000..5828b5d --- /dev/null +++ b/tests/eval_test/run_evals.py @@ -0,0 +1,136 @@ +""" +Entry point for the Teradata MCP eval suite. + +Usage: + uv run python run_evals.py # all modules + uv run python run_evals.py --module base # one module + uv run python run_evals.py --module base --type ambiguous_selection + uv run python run_evals.py --verbose + uv run python run_evals.py --module base --with-description-overrides +""" + +from __future__ import annotations + +import argparse +import os +import subprocess +import sys + +from preflight import run_preflight +from agent.client import description_overrides_enabled, resolve_description_overrides_file +from judge.report import format_run_index, load_latest_pointer + + +CASE_TYPE_FILTERS = { + "happy_path": "happy", + "ambiguous_selection": "ambiguous", + "missing_parameter": "missing", + "multi_tool": "multi_tool", + "multi_turn": "clarify_then_call", +} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run Teradata MCP evals via deepeval + pytest") + parser.add_argument("--module", help="Run evals for a specific module only (base, sec, dba, ...)") + parser.add_argument( + "--type", + dest="case_type", + help=( + "Filter by case type (happy_path, ambiguous_selection, missing_parameter, " + "multi_tool, multi_turn). Matches substrings in pytest case IDs." + ), + ) + parser.add_argument("--verbose", "-v", action="store_true", help="Verbose pytest output") + parser.add_argument( + "--skip-preflight", + action="store_true", + help="Skip Teradata eval-table check (not recommended for live eval runs)", + ) + parser.add_argument( + "--with-description-overrides", + action="store_true", + help=( + "Patch tool descriptions from description_overrides.json before routing " + "(default: use live MCP server descriptions as baseline)" + ), + ) + parser.add_argument( + "--description-overrides-file", + help="Path to overrides JSON (enables overrides; default: description_overrides.json)", + ) + parser.add_argument( + "--run-label", + help="Optional label appended to the run directory name (e.g. after-tablelist-fix)", + ) + parser.add_argument( + "--list-runs", + action="store_true", + help="List recent eval runs from results/index.json and exit", + ) + args = parser.parse_args() + + if args.list_runs: + print(format_run_index()) + pointer = load_latest_pointer() + if pointer: + print("") + print(f"Latest run: {pointer.get('run_id')}") + print(f" dir: results/{pointer.get('run_dir')}") + print(f" summary: results/{pointer.get('summary_md')}") + sys.exit(0) + + if not args.skip_preflight: + run_preflight() + + os.environ["EVALS_RUN_MODULE"] = args.module or "all" + os.environ["EVALS_RUN_TYPE"] = args.case_type or "all" + if args.run_label: + os.environ["EVALS_RUN_LABEL"] = args.run_label + elif "EVALS_RUN_LABEL" in os.environ: + del os.environ["EVALS_RUN_LABEL"] + + if args.with_description_overrides or args.description_overrides_file: + os.environ["USE_DESCRIPTION_OVERRIDES"] = "1" + if args.description_overrides_file: + os.environ["DESCRIPTION_OVERRIDES_FILE"] = args.description_overrides_file + + cmd = ["deepeval", "test", "run", "tests/"] + + if args.module: + cmd += ["-k", f"test_{args.module}"] + + if args.case_type: + keyword = CASE_TYPE_FILTERS.get(args.case_type, args.case_type) + existing_k = next((cmd[i + 1] for i, c in enumerate(cmd) if c == "-k"), None) + if existing_k: + idx = cmd.index("-k") + cmd[idx + 1] = f"{existing_k} and {keyword}" + else: + cmd += ["-k", keyword] + + if args.verbose: + cmd.append("-v") + + if description_overrides_enabled(): + overrides_file = resolve_description_overrides_file() + print(f"Tool descriptions: overrides from {overrides_file}") + else: + print("Tool descriptions: live MCP server (baseline)") + + print(f"Running: {' '.join(cmd)}\n") + result = subprocess.run(cmd) + if result.returncode in {0, 1}: + pointer = load_latest_pointer() + if pointer: + print("") + print(f"Eval run: {pointer.get('run_id')}") + print(f"Run directory: results/{pointer.get('run_dir')}") + print(f"Summary: results/{pointer.get('summary_md')}") + print("Latest copy: results/latest_summary.md") + print("All runs: uv run python run_evals.py --list-runs") + sys.exit(result.returncode) + + +if __name__ == "__main__": + main() diff --git a/tests/eval_test/setup_test_data.py b/tests/eval_test/setup_test_data.py new file mode 100644 index 0000000..d0a2058 --- /dev/null +++ b/tests/eval_test/setup_test_data.py @@ -0,0 +1,132 @@ +""" +Creates the two eval tables in EVALS_DATABASE and populates them with sample data. + +evals_employees — used by: base, qlty (stats), plot (radar) +evals_orders — used by: base, qlty (missing/negative values), plot (line/pie/polar) + +Usage: + python setup_test_data.py + python setup_test_data.py --drop-first # drop and recreate if tables already exist +""" + +from __future__ import annotations + +import argparse +import os + +import teradatasql +from dotenv import load_dotenv + +load_dotenv() + +CREATE_EMPLOYEES = """ +CREATE TABLE {db}.evals_employees ( + employee_id INTEGER NOT NULL, + name VARCHAR(100) NOT NULL, + department VARCHAR(50) NOT NULL, + salary DECIMAL(10,2), + region VARCHAR(50) NOT NULL, + hire_date DATE NOT NULL, + manager_id INTEGER, + PRIMARY KEY (employee_id) +) +""" + +CREATE_ORDERS = """ +CREATE TABLE {db}.evals_orders ( + order_id INTEGER NOT NULL, + customer_name VARCHAR(100) NOT NULL, + order_date DATE NOT NULL, + ship_date DATE, + amount DECIMAL(10,2) NOT NULL, + product_category VARCHAR(50) NOT NULL, + quantity INTEGER NOT NULL, + PRIMARY KEY (order_id) +) +""" + +EMPLOYEES_DATA = [ + (1, "Alice Johnson", "Engineering", 95000.00, "North", "2019-03-15", None), + (2, "Bob Smith", "Sales", 72000.00, "South", "2020-07-01", 1), + (3, "Carol White", "HR", 68000.00, "East", "2018-11-20", 1), + (4, "David Brown", "Finance", 85000.00, "West", "2021-01-10", 1), + (5, "Eva Martinez", "Engineering", 102000.00,"North", "2017-06-05", 1), + (6, "Frank Lee", "Sales", 69500.00, "South", "2022-04-18", 2), + (7, "Grace Kim", "Marketing", 77000.00, "East", "2020-09-30", 1), + (8, "Henry Adams", "Finance", 91000.00, "West", "2019-12-01", 4), + (9, "Isla Thompson", "HR", None, "North", "2023-02-14", 3), + (10, "Jack Wilson", "Engineering", 88000.00, "South", "2016-08-22", 5), +] + +ORDERS_DATA = [ + (1001, "Acme Corp", "2024-01-05", "2024-01-08", 1250.00, "Electronics", 3), + (1002, "Beta Ltd", "2024-01-12", "2024-01-15", -200.00, "Clothing", 1), # refund + (1003, "Gamma Inc", "2024-02-03", None, 875.50, "Food", 5), # not shipped + (1004, "Delta Co", "2024-02-17", "2024-02-20", 3400.00, "Electronics", 2), + (1005, "Epsilon LLC", "2024-03-01", "2024-03-04", 560.00, "Books", 8), + (1006, "Zeta Partners", "2024-03-15", None, -150.00, "Clothing", 1), # refund, not shipped + (1007, "Eta Group", "2024-04-02", "2024-04-06", 2100.00, "Sports", 4), + (1008, "Theta Corp", "2024-04-20", "2024-04-23", 930.00, "Food", 6), + (1009, "Iota Inc", "2024-05-08", None, 1780.00, "Electronics", 1), # not shipped + (1010, "Kappa Ltd", "2024-05-25", "2024-05-28", 4200.00, "Sports", 7), +] + + +def get_connection(): + host = os.environ["TERADATA_HOST"] + user = os.environ["TERADATA_USER"] + password = os.environ["TERADATA_PASSWORD"] + return teradatasql.connect(host=host, user=user, password=password) + + +def drop_tables(cursor, db: str) -> None: + for table in ("evals_employees", "evals_orders"): + try: + cursor.execute(f"DROP TABLE {db}.{table}") + print(f" Dropped {db}.{table}") + except Exception: + pass # table didn't exist + + +def create_and_populate(cursor, db: str) -> None: + print(f"Creating {db}.evals_employees ...") + cursor.execute(CREATE_EMPLOYEES.format(db=db)) + + for row in EMPLOYEES_DATA: + cursor.execute( + f"INSERT INTO {db}.evals_employees VALUES (?,?,?,?,?,?,?)", [list(row)] + ) + print(f" Inserted {len(EMPLOYEES_DATA)} rows") + + print(f"Creating {db}.evals_orders ...") + cursor.execute(CREATE_ORDERS.format(db=db)) + + for row in ORDERS_DATA: + cursor.execute( + f"INSERT INTO {db}.evals_orders VALUES (?,?,?,?,?,?,?)", [list(row)] + ) + print(f" Inserted {len(ORDERS_DATA)} rows") + + +def main() -> None: + parser = argparse.ArgumentParser(description="Create eval test tables in Teradata") + parser.add_argument("--drop-first", action="store_true", help="Drop tables before recreating") + args = parser.parse_args() + + db = os.environ.get("EVALS_DATABASE", "").strip() + if not db: + raise SystemExit("EVALS_DATABASE env var is not set — check your .env file") + + print(f"Connecting to Teradata (host={os.environ.get('TERADATA_HOST')}) ...") + with get_connection() as con: + with con.cursor() as cur: + if args.drop_first: + print("Dropping existing tables ...") + drop_tables(cur, db) + create_and_populate(cur, db) + + print("\nSetup complete. Run evals with: python run_evals.py") + + +if __name__ == "__main__": + main() diff --git a/tests/eval_test/suggest_overrides.py b/tests/eval_test/suggest_overrides.py new file mode 100644 index 0000000..50719bd --- /dev/null +++ b/tests/eval_test/suggest_overrides.py @@ -0,0 +1,163 @@ +""" +Draft description_overrides.json entries from failed eval cases. + +Usage: + uv run python suggest_overrides.py + uv run python suggest_overrides.py --type missing_parameter + uv run python suggest_overrides.py --summary results/latest_summary.json + uv run python suggest_overrides.py --dry-run + uv run python suggest_overrides.py --apply + uv run python suggest_overrides.py --apply --dry-run +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from agent.client import DEFAULT_OVERRIDES_FILE +from judge.report import resolve_default_summary_path, resolve_suggestion_output_path +from judge.suggest_overrides import ( + DEFAULT_CASE_TYPES, + apply_suggestions_to_overrides, + generate_suggestions, +) + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Suggest description_overrides.json entries from failed eval cases", + ) + parser.add_argument( + "--summary", + type=Path, + default=None, + help="Eval summary JSON to read (default: latest run from results/latest.json)", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Where to write the draft (default: alongside the source run, plus results/suggested_overrides.json)", + ) + parser.add_argument( + "--overrides-file", + type=Path, + default=DEFAULT_OVERRIDES_FILE, + help="Existing overrides file to include as context (default: description_overrides.json)", + ) + parser.add_argument( + "--type", + dest="case_type", + help=( + "Case type(s) to process — comma-separated subset of " + "ambiguous_selection, happy_path, missing_parameter, multi_tool " + f"(default: all)" + ), + ) + parser.add_argument("--dry-run", action="store_true", help="Generate: print prompts only. Apply: preview merge only.") + parser.add_argument( + "--apply", + action="store_true", + help="Merge reviewed suggestions into description_overrides.json (replaces existing tool entries)", + ) + parser.add_argument( + "--suggestions", + type=Path, + default=None, + help="Draft to apply when using --apply (default: latest run draft or results/suggested_overrides.json)", + ) + parser.add_argument( + "--tools", + help="Comma-separated tool names to apply when using --apply (default: all suggestions in the draft)", + ) + args = parser.parse_args() + + summary_path = args.summary or resolve_default_summary_path() + output_path = args.output or resolve_suggestion_output_path(summary_path) + suggestions_path = args.suggestions or output_path + + if args.apply: + only_tools = None + if args.tools: + only_tools = {part.strip() for part in args.tools.split(",") if part.strip()} + try: + result = apply_suggestions_to_overrides( + suggestions_path=suggestions_path, + overrides_path=args.overrides_file, + only_tools=only_tools, + dry_run=args.dry_run, + ) + except FileNotFoundError as exc: + print(exc, file=sys.stderr) + sys.exit(1) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) + + if result.get("message"): + print(result["message"]) + sys.exit(0) + + applied = result.get("applied", []) + removed = result.get("removed", []) + target = result["overrides_path"] + if args.dry_run: + print(f"Dry run — would write {result['applied_count']} tool(s) → {target}") + else: + print(f"Applied {result['applied_count']} tool(s) → {target}") + if applied: + print(f" applied: {', '.join(applied)}") + if removed: + print(f" removed: {', '.join(removed)}") + if not args.dry_run: + print("Re-run evals with overrides:") + print(" uv run python run_evals.py --with-description-overrides") + sys.exit(0) + + try: + payload = generate_suggestions( + summary_path=summary_path, + output_path=output_path, + overrides_path=args.overrides_file, + case_type=args.case_type, + dry_run=args.dry_run, + ) + except FileNotFoundError as exc: + print(exc, file=sys.stderr) + sys.exit(1) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) + except RuntimeError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + sys.exit(1) + + if payload.get("message"): + print(payload["message"]) + if payload.get("skipped"): + for entry in payload["skipped"]: + print(f" skipped {entry['case_id']}: {entry['reason']}") + sys.exit(0) + + if args.dry_run: + print(json.dumps(payload, indent=2)) + sys.exit(0) + + suggestion_count = len(payload.get("suggestions", {})) + case_count = len(payload.get("cases", [])) + skipped_count = len(payload.get("skipped", [])) + print(f"Wrote {suggestion_count} tool suggestion(s) from {case_count} case(s) → {output_path}") + if output_path.name == "suggested_overrides.json" and output_path.parent.name != "results": + print(f"Also copied to results/suggested_overrides.json") + if skipped_count: + print(f"Skipped {skipped_count} case(s) — see 'skipped' in {output_path}") + print("Review the draft, then apply accepted entries:") + print(" uv run python suggest_overrides.py --apply") + print(" uv run python run_evals.py --with-description-overrides") + + +if __name__ == "__main__": + main() diff --git a/tests/eval_test/teardown_test_data.py b/tests/eval_test/teardown_test_data.py new file mode 100644 index 0000000..0d04081 --- /dev/null +++ b/tests/eval_test/teardown_test_data.py @@ -0,0 +1,43 @@ +""" +Drops the eval tables from EVALS_DATABASE. + +Usage: + python teardown_test_data.py +""" + +from __future__ import annotations + +import os + +import teradatasql +from dotenv import load_dotenv + +load_dotenv() + +TABLES = ["evals_employees", "evals_orders"] + + +def main() -> None: + db = os.environ.get("EVALS_DATABASE") + if not db: + raise SystemExit("EVALS_DATABASE env var is not set — check your .env file") + + print(f"Connecting to Teradata (host={os.environ.get('TERADATA_HOST')}) ...") + with teradatasql.connect( + host=os.environ["TERADATA_HOST"], + user=os.environ["TERADATA_USER"], + password=os.environ["TERADATA_PASSWORD"], + ) as con: + with con.cursor() as cur: + for table in TABLES: + try: + cur.execute(f"DROP TABLE {db}.{table}") + print(f" Dropped {db}.{table}") + except Exception as exc: + print(f" Skipped {db}.{table}: {exc}") + + print("Teardown complete.") + + +if __name__ == "__main__": + main() diff --git a/tests/eval_test/tests/__init__.py b/tests/eval_test/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/eval_test/tests/case_runner.py b/tests/eval_test/tests/case_runner.py new file mode 100644 index 0000000..feb6620 --- /dev/null +++ b/tests/eval_test/tests/case_runner.py @@ -0,0 +1,435 @@ +"""Run single-turn and shallow multi-turn eval cases.""" + +from __future__ import annotations + +import os +from typing import Any + +from deepeval.evaluate.configs import CacheConfig, DisplayConfig, ErrorConfig +from deepeval.evaluate.execute import execute_test_cases +from deepeval.test_case import LLMTestCase, ToolCall + +from agent.client import run_agent, run_agent_turns +from judge.checks import ToolCallRecord, run_deterministic_checks +from judge.metrics import clarification_metric, get_metrics, tool_correctness_metric +from judge.report import CaseEvalResult, build_recommendation, record_case_result + +MAX_TURNS = 7 + + +def _extract_exception_detail(exc: Exception) -> str: + """Extract detailed error message from exception, unwrapping ExceptionGroup if needed.""" + # For ExceptionGroup (TaskGroup errors), extract the first exception + if hasattr(exc, 'exceptions') and exc.exceptions: + # ExceptionGroup / TaskGroup + first_exc = exc.exceptions[0] + return _extract_exception_detail(first_exc) + # For regular exceptions, include traceback context if available + if hasattr(exc, '__cause__') and exc.__cause__: + return f"{type(exc).__name__}: {str(exc)} (caused by {type(exc.__cause__).__name__}: {str(exc.__cause__)})" + return f"{type(exc).__name__}: {str(exc)}" + + +def validate_multi_turn_case(case: dict) -> None: + """Validate a shallow multi-turn case schema.""" + turns = case.get("turns") + if turns is None: + return + + if not isinstance(turns, list): + raise ValueError(f"[{case.get('id')}] turns must be a list") + + if len(turns) < 2: + raise ValueError(f"[{case.get('id')}] multi-turn cases need at least 2 turns") + + if len(turns) > MAX_TURNS: + raise ValueError(f"[{case.get('id')}] multi-turn cases allow at most {MAX_TURNS} turns") + + for index, turn in enumerate(turns, start=1): + is_clarification = turn.get("expect") == "clarification" + has_tools = bool(turn.get("expected_tools")) + if is_clarification == has_tools: + raise ValueError( + f"[{case.get('id')}] turn {index} must set exactly one of " + "'expect': 'clarification' or non-empty 'expected_tools'", + ) + if "input" not in turn: + raise ValueError(f"[{case.get('id')}] turn {index} is missing 'input'") + + +def _to_tool_calls(records: list[ToolCallRecord]) -> list[ToolCall]: + return [ToolCall(name=tc.name, input_parameters=tc.input_parameters) for tc in records] + + +def _tool_dicts(records: list[ToolCallRecord]) -> list[dict[str, Any]]: + return [{"name": tc.name, "params": tc.input_parameters} for tc in records] + + +def _make_test_case( + *, + user_input: str, + response: str, + tools_called: list[ToolCallRecord], + expected_tools_raw: list[dict[str, Any]], +) -> LLMTestCase: + return LLMTestCase( + input=user_input, + actual_output=response, + tools_called=_to_tool_calls(tools_called), + expected_tools=[ + ToolCall(name=t["name"], input_parameters=t.get("params", {})) + for t in expected_tools_raw + ], + ) + + +def _evaluate_metrics(test_case: LLMTestCase, metrics) -> tuple[list[str], bool]: + test_result = execute_test_cases( + [test_case], + metrics, + error_config=ErrorConfig(ignore_errors=False, skip_on_missing_params=False), + display_config=DisplayConfig(verbose_mode=False, show_indicator=False), + cache_config=CacheConfig(write_cache=False, use_cache=False), + identifier="eval", + _use_bar_indicator=False, + _is_assert_test=True, + )[0] + + if test_result.success: + return [], True + + reasons: list[str] = [] + for metric_data in test_result.metrics_data or []: + if metric_data.error is not None or not metric_data.success: + detail = metric_data.reason or metric_data.error or "metric failed" + reasons.append(f"{metric_data.name}: {detail}") + return reasons, False + + +def _failure_result( + case: dict, + *, + case_input: str, + failure_stage: str, + failure_detail: str, + expected_tools: list[dict[str, Any]] | None = None, + actual_tools: list[dict[str, Any]] | None = None, + actual_output: str | None = None, + metric_reasons: list[str] | None = None, + turn_details: list[dict[str, Any]] | None = None, +) -> CaseEvalResult: + expected = expected_tools if expected_tools is not None else case.get("expected_tools", []) + metric_reasons = metric_reasons or [] + recommendation = build_recommendation( + case, + failure_stage=failure_stage, + failure_detail=failure_detail, + expected_tools=expected, + actual_tools=actual_tools, + metric_reasons=metric_reasons, + ) + return CaseEvalResult( + case_id=case.get("id", ""), + case_type=case.get("type", "happy_path"), + description=case.get("description", ""), + input=case_input, + expected_tools=expected, + passed=False, + failure_stage=failure_stage, + failure_detail=failure_detail, + actual_tools=actual_tools, + actual_output=actual_output, + metric_reasons=metric_reasons, + recommendation=recommendation, + turn_details=turn_details, + ) + + +def _success_result( + case: dict, + *, + case_input: str, + expected_tools: list[dict[str, Any]], + actual_tools: list[dict[str, Any]], + actual_output: str, + turn_details: list[dict[str, Any]] | None = None, +) -> CaseEvalResult: + return CaseEvalResult( + case_id=case.get("id", ""), + case_type=case.get("type", "happy_path"), + description=case.get("description", ""), + input=case_input, + expected_tools=expected_tools, + passed=True, + actual_tools=actual_tools, + actual_output=actual_output, + turn_details=turn_details, + ) + + +def run_single_turn_case(case: dict, bedrock_client, agent_model_id: str, judge_llm) -> CaseEvalResult: + """Run a single-turn case and return a structured result.""" + validate_multi_turn_case(case) + if "turns" in case: + raise ValueError(f"[{case.get('id')}] use run_eval_case() for multi-turn cases") + + try: + agent_result = run_agent( + prompt=case["input"], + model_id=agent_model_id, + bedrock_client=bedrock_client, + ) + except Exception as exc: + return _failure_result( + case, + case_input=case["input"], + failure_stage="agent", + failure_detail=_extract_exception_detail(exc), + ) + + raw_calls = [ + ToolCallRecord(name=tc.name, input_parameters=tc.input_parameters) + for tc in agent_result.tool_calls + ] + actual_tools = _tool_dicts(raw_calls) + det_errors = run_deterministic_checks(case, raw_calls) + if det_errors: + return _failure_result( + case, + case_input=case["input"], + failure_stage="deterministic", + failure_detail="; ".join(det_errors), + actual_tools=actual_tools, + actual_output=agent_result.final_response, + ) + + test_case = _make_test_case( + user_input=case["input"], + response=agent_result.final_response, + tools_called=raw_calls, + expected_tools_raw=case.get("expected_tools", []), + ) + metric_reasons, passed = _evaluate_metrics(test_case, get_metrics(case, judge_llm)) + if not passed: + return _failure_result( + case, + case_input=case["input"], + failure_stage="metric", + failure_detail="; ".join(metric_reasons), + actual_tools=actual_tools, + actual_output=agent_result.final_response, + metric_reasons=metric_reasons, + ) + + return _success_result( + case, + case_input=case["input"], + expected_tools=case.get("expected_tools", []), + actual_tools=actual_tools, + actual_output=agent_result.final_response, + ) + + +def run_multi_turn_case(case: dict, bedrock_client, agent_model_id: str, judge_llm) -> CaseEvalResult: + """Run and score a shallow multi-turn case (2–7 turns).""" + validate_multi_turn_case(case) + turns = case["turns"] + prompts = [turn["input"] for turn in turns] + case_input = " | ".join(f"Turn {index}: {turn['input']}" for index, turn in enumerate(turns, start=1)) + + max_steps_per_turn = int(os.environ.get("AGENT_MAX_STEPS_PER_TURN", "3")) + try: + turn_results = run_agent_turns( + prompts=prompts, + model_id=agent_model_id, + bedrock_client=bedrock_client, + max_steps_per_turn=max_steps_per_turn, + ) + except Exception as exc: + return _failure_result( + case, + case_input=case_input, + failure_stage="agent", + failure_detail=_extract_exception_detail(exc), + expected_tools=[], + ) + + conversation_prefix = "" + turn_details: list[dict[str, Any]] = [] + + for turn_number, (turn_spec, turn_result) in enumerate(zip(turns, turn_results, strict=True), start=1): + raw_calls = [ + ToolCallRecord(name=tc.name, input_parameters=tc.input_parameters) + for tc in turn_result.tool_calls + ] + actual_tools = _tool_dicts(raw_calls) + turn_label = f"{case.get('id')} turn {turn_number}" + turn_input = f"{conversation_prefix}User: {turn_spec['input']}" + + if turn_spec.get("expect") == "clarification": + pseudo_case = {"id": turn_label, "type": "missing_parameter", "expected_tools": []} + det_errors = run_deterministic_checks(pseudo_case, raw_calls) + if det_errors: + turn_details.append( + { + "turn": turn_number, + "input": turn_spec["input"], + "mode": "clarification", + "passed": False, + "failure_stage": "deterministic", + "failure_detail": "; ".join(det_errors), + "actual_tools": actual_tools, + } + ) + return _failure_result( + case, + case_input=case_input, + failure_stage="deterministic", + failure_detail=f"turn {turn_number}: {'; '.join(det_errors)}", + expected_tools=[], + actual_tools=actual_tools, + actual_output=turn_result.final_response, + turn_details=turn_details, + ) + + test_case = _make_test_case( + user_input=turn_input, + response=turn_result.final_response, + tools_called=[], + expected_tools_raw=[], + ) + metric_reasons, passed = _evaluate_metrics(test_case, [clarification_metric(judge_llm)]) + else: + expected_tools = turn_spec.get("expected_tools", []) + pseudo_case = { + "id": turn_label, + "type": "happy_path", + "expected_tools": expected_tools, + } + det_errors = run_deterministic_checks(pseudo_case, raw_calls) + if det_errors: + turn_details.append( + { + "turn": turn_number, + "input": turn_spec["input"], + "mode": "tool", + "passed": False, + "failure_stage": "deterministic", + "failure_detail": "; ".join(det_errors), + "expected_tools": expected_tools, + "actual_tools": actual_tools, + } + ) + return _failure_result( + case, + case_input=case_input, + failure_stage="deterministic", + failure_detail=f"turn {turn_number}: {'; '.join(det_errors)}", + expected_tools=expected_tools, + actual_tools=actual_tools, + actual_output=turn_result.final_response, + turn_details=turn_details, + ) + + test_case = _make_test_case( + user_input=turn_input, + response=turn_result.final_response, + tools_called=raw_calls, + expected_tools_raw=expected_tools, + ) + metric_reasons, passed = _evaluate_metrics(test_case, [tool_correctness_metric(judge_llm)]) + + if not passed: + turn_details.append( + { + "turn": turn_number, + "input": turn_spec["input"], + "mode": "clarification" if turn_spec.get("expect") == "clarification" else "tool", + "passed": False, + "failure_stage": "metric", + "failure_detail": "; ".join(metric_reasons), + "expected_tools": turn_spec.get("expected_tools", []), + "actual_tools": actual_tools, + } + ) + return _failure_result( + case, + case_input=case_input, + failure_stage="metric", + failure_detail=f"turn {turn_number}: {'; '.join(metric_reasons)}", + expected_tools=turn_spec.get("expected_tools", []), + actual_tools=actual_tools, + actual_output=turn_result.final_response, + metric_reasons=metric_reasons, + turn_details=turn_details, + ) + + turn_details.append( + { + "turn": turn_number, + "input": turn_spec["input"], + "mode": "clarification" if turn_spec.get("expect") == "clarification" else "tool", + "passed": True, + "actual_tools": actual_tools, + } + ) + conversation_prefix += f"User: {turn_spec['input']}\nAssistant: {turn_result.final_response}\n" + + last_turn = turn_results[-1] + last_tools = [ + ToolCallRecord(name=tc.name, input_parameters=tc.input_parameters) + for tc in last_turn.tool_calls + ] + return _success_result( + case, + case_input=case_input, + expected_tools=turns[-1].get("expected_tools", []), + actual_tools=_tool_dicts(last_tools), + actual_output=last_turn.final_response, + turn_details=turn_details, + ) + + +def run_eval_case(case: dict, bedrock_client, agent_model_id: str, judge_llm) -> CaseEvalResult: + """Run any eval case and return a structured result.""" + validate_multi_turn_case(case) + if "turns" in case: + return run_multi_turn_case(case, bedrock_client, agent_model_id, judge_llm) + return run_single_turn_case(case, bedrock_client, agent_model_id, judge_llm) + + +def build_test_case(case: dict, bedrock_client, agent_model_id: str) -> LLMTestCase: + """Run a single-turn case and return a deepeval LLMTestCase (metrics not scored).""" + validate_multi_turn_case(case) + if "turns" in case: + raise ValueError(f"[{case.get('id')}] use assert_eval_case() for multi-turn cases") + + agent_result = run_agent( + prompt=case["input"], + model_id=agent_model_id, + bedrock_client=bedrock_client, + ) + raw_calls = [ + ToolCallRecord(name=tc.name, input_parameters=tc.input_parameters) + for tc in agent_result.tool_calls + ] + det_errors = run_deterministic_checks(case, raw_calls) + if det_errors: + case_id = case.get("id", "") + raise AssertionError(f"[{case_id}] deterministic check failed: {'; '.join(det_errors)}") + + return _make_test_case( + user_input=case["input"], + response=agent_result.final_response, + tools_called=raw_calls, + expected_tools_raw=case.get("expected_tools", []), + ) + + +def assert_eval_case(case: dict, bedrock_client, agent_model_id: str, judge_llm) -> None: + """Run and score any eval case (single- or multi-turn).""" + result = run_eval_case(case, bedrock_client, agent_model_id, judge_llm) + record_case_result(result) + if not result.passed: + detail = result.failure_detail or "; ".join(result.metric_reasons) or "eval case failed" + raise AssertionError(f"[{result.case_id}] {result.failure_stage} check failed: {detail}") diff --git a/tests/eval_test/tests/conftest.py b/tests/eval_test/tests/conftest.py new file mode 100644 index 0000000..ae44f8b --- /dev/null +++ b/tests/eval_test/tests/conftest.py @@ -0,0 +1,107 @@ +"""Shared fixtures and helpers for the Teradata MCP eval suite.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import pytest +from dotenv import load_dotenv + +load_dotenv() + +CASES_DIR = Path(__file__).parent.parent / "cases" +MODULES = ["base", "dba", "sec", "qlty", "chat", "plot", "tmpl"] + + +def _substitute(obj, evals_db: str): + """Recursively replace {EVALS_DATABASE} in strings within dicts/lists.""" + if isinstance(obj, str): + return obj.replace("{EVALS_DATABASE}", evals_db) + if isinstance(obj, dict): + return {k: _substitute(v, evals_db) for k, v in obj.items()} + if isinstance(obj, list): + return [_substitute(i, evals_db) for i in obj] + return obj + + +def load_cases(module: str) -> list[dict]: + path = CASES_DIR / f"{module}.json" + if not path.exists(): + return [] + data = json.loads(path.read_text()) + # Filter out comment/instruction stubs + return [c for c in data.get("cases", []) if "id" in c] + + +def build_test_case(case: dict, bedrock_client, agent_model_id: str): + """Run a single-turn case and return a deepeval LLMTestCase.""" + from tests.case_runner import build_test_case as _build_test_case + + evals_db = os.environ.get("EVALS_DATABASE", "").strip() + resolved = _substitute(case, evals_db) + return _build_test_case(resolved, bedrock_client, agent_model_id) + + +def assert_eval_case(case: dict, bedrock_client, agent_model_id: str, judge_llm) -> None: + """Run and score any eval case (single- or multi-turn).""" + from tests.case_runner import assert_eval_case as _assert_eval_case + + evals_db = os.environ.get("EVALS_DATABASE", "").strip() + resolved = _substitute(case, evals_db) + _assert_eval_case(resolved, bedrock_client, agent_model_id, judge_llm) + + +@pytest.fixture(scope="session") +def bedrock_client(): + import boto3 + region = os.environ.get("AWS_REGION", "us-east-1") + return boto3.client("bedrock-runtime", region_name=region) + + +@pytest.fixture(scope="session") +def agent_model_id() -> str: + return os.environ.get("BEDROCK_MODEL_ID", "anthropic.claude-3-5-sonnet-20241022-v2:0") + + +@pytest.fixture(scope="session") +def judge_llm(bedrock_client): + from judge.bedrock_llm import BedrockLLM + return BedrockLLM(bedrock_client=bedrock_client) + + +def pytest_sessionstart(session) -> None: + """Initialize eval result collection for live eval runs.""" + from agent.client import get_description_override_status + from judge.report import begin_eval_run + + agent_model_id = os.environ.get("BEDROCK_MODEL_ID", "anthropic.claude-3-5-sonnet-20241022-v2:0") + judge_model_id = os.environ.get("BEDROCK_JUDGE_MODEL_ID", agent_model_id) + override_status = get_description_override_status() + begin_eval_run( + agent_model_id=agent_model_id, + judge_model_id=judge_model_id, + evals_database=os.environ.get("EVALS_DATABASE", "").strip(), + description_mode=str(override_status["mode"]), + description_overrides_file=override_status.get("file"), # type: ignore[arg-type] + description_override_count=int(override_status.get("tool_count") or 0), + ) + + +def pytest_sessionfinish(session, exitstatus) -> None: + """Write a markdown/json summary when live eval cases were executed.""" + from judge.report import get_current_report, write_eval_summary + + report = get_current_report() + if report is None or not report.results: + return + + artifacts = write_eval_summary(report) + terminal = session.config.pluginmanager.get_plugin("terminalreporter") + if terminal is not None: + terminal.write_line("") + terminal.write_line(f"Eval run: {artifacts.run_id}") + terminal.write_line(f"Run directory: results/{artifacts.run_dir.name}") + terminal.write_line("Summary: results/latest_summary.md (copy of this run)") + terminal.write_line("Index: results/index.json") diff --git a/tests/eval_test/uv.lock b/tests/eval_test/uv.lock new file mode 100644 index 0000000..d6cdb10 --- /dev/null +++ b/tests/eval_test/uv.lock @@ -0,0 +1,2101 @@ +version = 1 +revision = 1 +requires-python = ">=3.11" + +[[package]] +name = "aiohappyeyeballs" +version = "2.6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/c6/61a2d7b7572279226bb2e7f61d7a19ca7c90da0329c93fa0d560cbf288d8/aiohappyeyeballs-2.6.2.tar.gz", hash = "sha256:e202810ee718bd01fc6ef49e8ea53d023d5cb6b581076d7925aa499fa55dbe64", size = 22591 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/fc/a7bf5b6e4e617b45f90f2d9d2a68519c249c81dd4fc2658c7a2a61c4f4b7/aiohappyeyeballs-2.6.2-py3-none-any.whl", hash = "sha256:4708045e2d7a6c6bdf8aafa8ed39649eaf926a4543b54560659129e3365953c4", size = 15062 }, +] + +[[package]] +name = "aiohttp" +version = "3.13.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohappyeyeballs" }, + { name = "aiosignal" }, + { name = "attrs" }, + { name = "frozenlist" }, + { name = "multidict" }, + { name = "propcache" }, + { name = "yarl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/9a/152096d4808df8e4268befa55fba462f440f14beab85e8ad9bf990516918/aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1", size = 7858271 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/f5/a20c4ac64aeaef1679e25c9983573618ff765d7aa829fa2b84ae7573169e/aiohttp-3.13.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ab7229b6f9b5c1ba4910d6c41a9eb11f543eadb3f384df1b4c293f4e73d44d6", size = 757513 }, + { url = "https://files.pythonhosted.org/packages/75/0a/39fa6c6b179b53fcb3e4b3d2b6d6cad0180854eda17060c7218540102bef/aiohttp-3.13.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8f14c50708bb156b3a3ca7230b3d820199d56a48e3af76fa21c2d6087190fe3d", size = 506748 }, + { url = "https://files.pythonhosted.org/packages/87/ec/e38ce072e724fd7add6243613f8d1810da084f54175353d25ccf9f9c7e5a/aiohttp-3.13.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e7d2f8616f0ff60bd332022279011776c3ac0faa0f1b463f7bb12326fbc97a1c", size = 501673 }, + { url = "https://files.pythonhosted.org/packages/ba/ba/3bc7525d7e2beaa11b309a70d48b0d3cfc3c2089ec6a7d0820d59c657053/aiohttp-3.13.5-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a2567b72e1ffc3ab25510db43f355b29eeada56c0a622e58dcdb19530eb0a3cb", size = 1763757 }, + { url = "https://files.pythonhosted.org/packages/5e/ab/e87744cf18f1bd78263aba24924d4953b41086bd3a31d22452378e9028a0/aiohttp-3.13.5-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:fb0540c854ac9c0c5ad495908fdfd3e332d553ec731698c0e29b1877ba0d2ec6", size = 1720152 }, + { url = "https://files.pythonhosted.org/packages/6b/f3/ed17a6f2d742af17b50bae2d152315ed1b164b07a5fd5cc1754d99e4dfa5/aiohttp-3.13.5-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c9883051c6972f58bfc4ebb2116345ee2aa151178e99c3f2b2bbe2af712abd13", size = 1818010 }, + { url = "https://files.pythonhosted.org/packages/53/06/ecbc63dc937192e2a5cb46df4d3edb21deb8225535818802f210a6ea5816/aiohttp-3.13.5-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2294172ce08a82fb7c7273485895de1fa1186cc8294cfeb6aef4af42ad261174", size = 1907251 }, + { url = "https://files.pythonhosted.org/packages/7e/a5/0521aa32c1ddf3aa1e71dcc466be0b7db2771907a13f18cddaa45967d97b/aiohttp-3.13.5-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3a807cabd5115fb55af198b98178997a5e0e57dead43eb74a93d9c07d6d4a7dc", size = 1759969 }, + { url = "https://files.pythonhosted.org/packages/f6/78/a38f8c9105199dd3b9706745865a8a59d0041b6be0ca0cc4b2ccf1bab374/aiohttp-3.13.5-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa6d0d932e0f39c02b80744273cd5c388a2d9bc07760a03164f229c8e02662f6", size = 1616871 }, + { url = "https://files.pythonhosted.org/packages/6f/41/27392a61ead8ab38072105c71aa44ff891e71653fe53d576a7067da2b4e8/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:60869c7ac4aaabe7110f26499f3e6e5696eae98144735b12a9c3d9eae2b51a49", size = 1739844 }, + { url = "https://files.pythonhosted.org/packages/6e/55/5564e7ae26d94f3214250009a0b1c65a0c6af4bf88924ccb6fdab901de28/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:26d2f8546f1dfa75efa50c3488215a903c0168d253b75fba4210f57ab77a0fb8", size = 1731969 }, + { url = "https://files.pythonhosted.org/packages/6d/c5/705a3929149865fc941bcbdd1047b238e4a72bcb215a9b16b9d7a2e8d992/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f1162a1492032c82f14271e831c8f4b49f2b6078f4f5fc74de2c912fa225d51d", size = 1795193 }, + { url = "https://files.pythonhosted.org/packages/a6/19/edabed62f718d02cff7231ca0db4ef1c72504235bc467f7b67adb1679f48/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:8b14eb3262fad0dc2f89c1a43b13727e709504972186ff6a99a3ecaa77102b6c", size = 1606477 }, + { url = "https://files.pythonhosted.org/packages/de/fc/76f80ef008675637d88d0b21584596dc27410a990b0918cb1e5776545b5b/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:ca9ac61ac6db4eb6c2a0cd1d0f7e1357647b638ccc92f7e9d8d133e71ed3c6ac", size = 1813198 }, + { url = "https://files.pythonhosted.org/packages/e5/67/5b3ac26b80adb20ea541c487f73730dc8fa107d632c998f25bbbab98fcda/aiohttp-3.13.5-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7996023b2ed59489ae4762256c8516df9820f751cf2c5da8ed2fb20ee50abab3", size = 1752321 }, + { url = "https://files.pythonhosted.org/packages/88/06/e4a2e49255ea23fa4feeb5ab092d90240d927c15e47b5b5c48dff5a9ce29/aiohttp-3.13.5-cp311-cp311-win32.whl", hash = "sha256:77dfa48c9f8013271011e51c00f8ada19851f013cde2c48fca1ba5e0caf5bb06", size = 439069 }, + { url = "https://files.pythonhosted.org/packages/c0/43/8c7163a596dab4f8be12c190cf467a1e07e4734cf90eebb39f7f5d53fc6a/aiohttp-3.13.5-cp311-cp311-win_amd64.whl", hash = "sha256:d3a4834f221061624b8887090637db9ad4f61752001eae37d56c52fddade2dc8", size = 462859 }, + { url = "https://files.pythonhosted.org/packages/be/6f/353954c29e7dcce7cf00280a02c75f30e133c00793c7a2ed3776d7b2f426/aiohttp-3.13.5-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:023ecba036ddd840b0b19bf195bfae970083fd7024ce1ac22e9bba90464620e9", size = 748876 }, + { url = "https://files.pythonhosted.org/packages/f5/1b/428a7c64687b3b2e9cd293186695affc0e1e54a445d0361743b231f11066/aiohttp-3.13.5-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:15c933ad7920b7d9a20de151efcd05a6e38302cbf0e10c9b2acb9a42210a2416", size = 499557 }, + { url = "https://files.pythonhosted.org/packages/29/47/7be41556bfbb6917069d6a6634bb7dd5e163ba445b783a90d40f5ac7e3a7/aiohttp-3.13.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ab2899f9fa2f9f741896ebb6fa07c4c883bfa5c7f2ddd8cf2aafa86fa981b2d2", size = 500258 }, + { url = "https://files.pythonhosted.org/packages/67/84/c9ecc5828cb0b3695856c07c0a6817a99d51e2473400f705275a2b3d9239/aiohttp-3.13.5-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a60eaa2d440cd4707696b52e40ed3e2b0f73f65be07fd0ef23b6b539c9c0b0b4", size = 1749199 }, + { url = "https://files.pythonhosted.org/packages/f0/d3/3c6d610e66b495657622edb6ae7c7fd31b2e9086b4ec50b47897ad6042a9/aiohttp-3.13.5-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:55b3bdd3292283295774ab585160c4004f4f2f203946997f49aac032c84649e9", size = 1721013 }, + { url = "https://files.pythonhosted.org/packages/49/a0/24409c12217456df0bae7babe3b014e460b0b38a8e60753d6cb339f6556d/aiohttp-3.13.5-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c2b2355dc094e5f7d45a7bb262fe7207aa0460b37a0d87027dcf21b5d890e7d5", size = 1781501 }, + { url = "https://files.pythonhosted.org/packages/98/9d/b65ec649adc5bccc008b0957a9a9c691070aeac4e41cea18559fef49958b/aiohttp-3.13.5-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b38765950832f7d728297689ad78f5f2cf79ff82487131c4d26fe6ceecdc5f8e", size = 1878981 }, + { url = "https://files.pythonhosted.org/packages/57/d8/8d44036d7eb7b6a8ec4c5494ea0c8c8b94fbc0ed3991c1a7adf230df03bf/aiohttp-3.13.5-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b18f31b80d5a33661e08c89e202edabf1986e9b49c42b4504371daeaa11b47c1", size = 1767934 }, + { url = "https://files.pythonhosted.org/packages/31/04/d3f8211f273356f158e3464e9e45484d3fb8c4ce5eb2f6fe9405c3273983/aiohttp-3.13.5-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:33add2463dde55c4f2d9635c6ab33ce154e5ecf322bd26d09af95c5f81cfa286", size = 1566671 }, + { url = "https://files.pythonhosted.org/packages/41/db/073e4ebe00b78e2dfcacff734291651729a62953b48933d765dc513bf798/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:327cc432fdf1356fb4fbc6fe833ad4e9f6aacb71a8acaa5f1855e4b25910e4a9", size = 1705219 }, + { url = "https://files.pythonhosted.org/packages/48/45/7dfba71a2f9fd97b15c95c06819de7eb38113d2cdb6319669195a7d64270/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:7c35b0bf0b48a70b4cb4fc5d7bed9b932532728e124874355de1a0af8ec4bc88", size = 1743049 }, + { url = "https://files.pythonhosted.org/packages/18/71/901db0061e0f717d226386a7f471bb59b19566f2cae5f0d93874b017271f/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:df23d57718f24badef8656c49743e11a89fd6f5358fa8a7b96e728fda2abf7d3", size = 1749557 }, + { url = "https://files.pythonhosted.org/packages/08/d5/41eebd16066e59cd43728fe74bce953d7402f2b4ddfdfef2c0e9f17ca274/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:02e048037a6501a5ec1f6fc9736135aec6eb8a004ce48838cb951c515f32c80b", size = 1558931 }, + { url = "https://files.pythonhosted.org/packages/30/e6/4a799798bf05740e66c3a1161079bda7a3dd8e22ca392481d7a7f9af82a6/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:31cebae8b26f8a615d2b546fee45d5ffb76852ae6450e2a03f42c9102260d6fe", size = 1774125 }, + { url = "https://files.pythonhosted.org/packages/84/63/7749337c90f92bc2cb18f9560d67aa6258c7060d1397d21529b8004fcf6f/aiohttp-3.13.5-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:888e78eb5ca55a615d285c3c09a7a91b42e9dd6fc699b166ebd5dee87c9ccf14", size = 1732427 }, + { url = "https://files.pythonhosted.org/packages/98/de/cf2f44ff98d307e72fb97d5f5bbae3bfcb442f0ea9790c0bf5c5c2331404/aiohttp-3.13.5-cp312-cp312-win32.whl", hash = "sha256:8bd3ec6376e68a41f9f95f5ed170e2fcf22d4eb27a1f8cb361d0508f6e0557f3", size = 433534 }, + { url = "https://files.pythonhosted.org/packages/aa/ca/eadf6f9c8fa5e31d40993e3db153fb5ed0b11008ad5d9de98a95045bed84/aiohttp-3.13.5-cp312-cp312-win_amd64.whl", hash = "sha256:110e448e02c729bcebb18c60b9214a87ba33bac4a9fa5e9a5f139938b56c6cb1", size = 460446 }, + { url = "https://files.pythonhosted.org/packages/78/e9/d76bf503005709e390122d34e15256b88f7008e246c4bdbe915cd4f1adce/aiohttp-3.13.5-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:a5029cc80718bbd545123cd8fe5d15025eccaaaace5d0eeec6bd556ad6163d61", size = 742930 }, + { url = "https://files.pythonhosted.org/packages/57/00/4b7b70223deaebd9bb85984d01a764b0d7bd6526fcdc73cca83bcbe7243e/aiohttp-3.13.5-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4bb6bf5811620003614076bdc807ef3b5e38244f9d25ca5fe888eaccea2a9832", size = 496927 }, + { url = "https://files.pythonhosted.org/packages/9c/f5/0fb20fb49f8efdcdce6cd8127604ad2c503e754a8f139f5e02b01626523f/aiohttp-3.13.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a84792f8631bf5a94e52d9cc881c0b824ab42717165a5579c760b830d9392ac9", size = 497141 }, + { url = "https://files.pythonhosted.org/packages/3b/86/b7c870053e36a94e8951b803cb5b909bfbc9b90ca941527f5fcafbf6b0fa/aiohttp-3.13.5-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:57653eac22c6a4c13eb22ecf4d673d64a12f266e72785ab1c8b8e5940d0e8090", size = 1732476 }, + { url = "https://files.pythonhosted.org/packages/b5/e5/4e161f84f98d80c03a238671b4136e6530453d65262867d989bbe78244d0/aiohttp-3.13.5-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e5e5f7debc7a57af53fdf5c5009f9391d9f4c12867049d509bf7bb164a6e295b", size = 1706507 }, + { url = "https://files.pythonhosted.org/packages/d4/56/ea11a9f01518bd5a2a2fcee869d248c4b8a0cfa0bb13401574fa31adf4d4/aiohttp-3.13.5-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c719f65bebcdf6716f10e9eff80d27567f7892d8988c06de12bbbd39307c6e3a", size = 1773465 }, + { url = "https://files.pythonhosted.org/packages/eb/40/333ca27fb74b0383f17c90570c748f7582501507307350a79d9f9f3c6eb1/aiohttp-3.13.5-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d97f93fdae594d886c5a866636397e2bcab146fd7a132fd6bb9ce182224452f8", size = 1873523 }, + { url = "https://files.pythonhosted.org/packages/f0/d2/e2f77eef1acb7111405433c707dc735e63f67a56e176e72e9e7a2cd3f493/aiohttp-3.13.5-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3df334e39d4c2f899a914f1dba283c1aadc311790733f705182998c6f7cae665", size = 1754113 }, + { url = "https://files.pythonhosted.org/packages/fb/56/3f653d7f53c89669301ec9e42c95233e2a0c0a6dd051269e6e678db4fdb0/aiohttp-3.13.5-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fe6970addfea9e5e081401bcbadf865d2b6da045472f58af08427e108d618540", size = 1562351 }, + { url = "https://files.pythonhosted.org/packages/ec/a6/9b3e91eb8ae791cce4ee736da02211c85c6f835f1bdfac0594a8a3b7018c/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7becdf835feff2f4f335d7477f121af787e3504b48b449ff737afb35869ba7bb", size = 1693205 }, + { url = "https://files.pythonhosted.org/packages/98/fc/bfb437a99a2fcebd6b6eaec609571954de2ed424f01c352f4b5504371dd3/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:676e5651705ad5d8a70aeb8eb6936c436d8ebbd56e63436cb7dd9bb36d2a9a46", size = 1730618 }, + { url = "https://files.pythonhosted.org/packages/e4/b6/c8534862126191a034f68153194c389addc285a0f1347d85096d349bbc15/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:9b16c653d38eb1a611cc898c41e76859ca27f119d25b53c12875fd0474ae31a8", size = 1745185 }, + { url = "https://files.pythonhosted.org/packages/0b/93/4ca8ee2ef5236e2707e0fd5fecb10ce214aee1ff4ab307af9c558bda3b37/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:999802d5fa0389f58decd24b537c54aa63c01c3219ce17d1214cbda3c2b22d2d", size = 1557311 }, + { url = "https://files.pythonhosted.org/packages/57/ae/76177b15f18c5f5d094f19901d284025db28eccc5ae374d1d254181d33f4/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:ec707059ee75732b1ba130ed5f9580fe10ff75180c812bc267ded039db5128c6", size = 1773147 }, + { url = "https://files.pythonhosted.org/packages/01/a4/62f05a0a98d88af59d93b7fcac564e5f18f513cb7471696ac286db970d6a/aiohttp-3.13.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2d6d44a5b48132053c2f6cd5c8cb14bc67e99a63594e336b0f2af81e94d5530c", size = 1730356 }, + { url = "https://files.pythonhosted.org/packages/e4/85/fc8601f59dfa8c9523808281f2da571f8b4699685f9809a228adcc90838d/aiohttp-3.13.5-cp313-cp313-win32.whl", hash = "sha256:329f292ed14d38a6c4c435e465f48bebb47479fd676a0411936cc371643225cc", size = 432637 }, + { url = "https://files.pythonhosted.org/packages/c0/1b/ac685a8882896acf0f6b31d689e3792199cfe7aba37969fa91da63a7fa27/aiohttp-3.13.5-cp313-cp313-win_amd64.whl", hash = "sha256:69f571de7500e0557801c0b51f4780482c0ec5fe2ac851af5a92cfce1af1cb83", size = 458896 }, + { url = "https://files.pythonhosted.org/packages/5d/ce/46572759afc859e867a5bc8ec3487315869013f59281ce61764f76d879de/aiohttp-3.13.5-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:eb4639f32fd4a9904ab8fb45bf3383ba71137f3d9d4ba25b3b3f3109977c5b8c", size = 745721 }, + { url = "https://files.pythonhosted.org/packages/13/fe/8a2efd7626dbe6049b2ef8ace18ffda8a4dfcbe1bcff3ac30c0c7575c20b/aiohttp-3.13.5-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:7e5dc4311bd5ac493886c63cbf76ab579dbe4641268e7c74e48e774c74b6f2be", size = 497663 }, + { url = "https://files.pythonhosted.org/packages/9b/91/cc8cc78a111826c54743d88651e1687008133c37e5ee615fee9b57990fac/aiohttp-3.13.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:756c3c304d394977519824449600adaf2be0ccee76d206ee339c5e76b70ded25", size = 499094 }, + { url = "https://files.pythonhosted.org/packages/0a/33/a8362cb15cf16a3af7e86ed11962d5cd7d59b449202dc576cdc731310bde/aiohttp-3.13.5-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecc26751323224cf8186efcf7fbcbc30f4e1d8c7970659daf25ad995e4032a56", size = 1726701 }, + { url = "https://files.pythonhosted.org/packages/45/0c/c091ac5c3a17114bd76cbf85d674650969ddf93387876cf67f754204bd77/aiohttp-3.13.5-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:10a75acfcf794edf9d8db50e5a7ec5fc818b2a8d3f591ce93bc7b1210df016d2", size = 1683360 }, + { url = "https://files.pythonhosted.org/packages/23/73/bcee1c2b79bc275e964d1446c55c54441a461938e70267c86afaae6fba27/aiohttp-3.13.5-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0f7a18f258d124cd678c5fe072fe4432a4d5232b0657fca7c1847f599233c83a", size = 1773023 }, + { url = "https://files.pythonhosted.org/packages/c7/ef/720e639df03004fee2d869f771799d8c23046dec47d5b81e396c7cda583a/aiohttp-3.13.5-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:df6104c009713d3a89621096f3e3e88cc323fd269dbd7c20afe18535094320be", size = 1853795 }, + { url = "https://files.pythonhosted.org/packages/bd/c9/989f4034fb46841208de7aeeac2c6d8300745ab4f28c42f629ba77c2d916/aiohttp-3.13.5-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:241a94f7de7c0c3b616627aaad530fe2cb620084a8b144d3be7b6ecfe95bae3b", size = 1730405 }, + { url = "https://files.pythonhosted.org/packages/ce/75/ee1fd286ca7dc599d824b5651dad7b3be7ff8d9a7e7b3fe9820d9180f7db/aiohttp-3.13.5-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c974fb66180e58709b6fc402846f13791240d180b74de81d23913abe48e96d94", size = 1558082 }, + { url = "https://files.pythonhosted.org/packages/c3/20/1e9e6650dfc436340116b7aa89ff8cb2bbdf0abc11dfaceaad8f74273a10/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:6e27ea05d184afac78aabbac667450c75e54e35f62238d44463131bd3f96753d", size = 1692346 }, + { url = "https://files.pythonhosted.org/packages/d8/40/8ebc6658d48ea630ac7903912fe0dd4e262f0e16825aa4c833c56c9f1f56/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a79a6d399cef33a11b6f004c67bb07741d91f2be01b8d712d52c75711b1e07c7", size = 1698891 }, + { url = "https://files.pythonhosted.org/packages/d8/78/ea0ae5ec8ba7a5c10bdd6e318f1ba5e76fcde17db8275188772afc7917a4/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:c632ce9c0b534fbe25b52c974515ed674937c5b99f549a92127c85f771a78772", size = 1742113 }, + { url = "https://files.pythonhosted.org/packages/8a/66/9d308ed71e3f2491be1acb8769d96c6f0c47d92099f3bc9119cada27b357/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fceedde51fbd67ee2bcc8c0b33d0126cc8b51ef3bbde2f86662bd6d5a6f10ec5", size = 1553088 }, + { url = "https://files.pythonhosted.org/packages/da/a6/6cc25ed8dfc6e00c90f5c6d126a98e2cf28957ad06fa1036bd34b6f24a2c/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:f92995dfec9420bb69ae629abf422e516923ba79ba4403bc750d94fb4a6c68c1", size = 1757976 }, + { url = "https://files.pythonhosted.org/packages/c1/2b/cce5b0ffe0de99c83e5e36d8f828e4161e415660a9f3e58339d07cce3006/aiohttp-3.13.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:20ae0ff08b1f2c8788d6fb85afcb798654ae6ba0b747575f8562de738078457b", size = 1712444 }, + { url = "https://files.pythonhosted.org/packages/6c/cf/9e1795b4160c58d29421eafd1a69c6ce351e2f7c8d3c6b7e4ca44aea1a5b/aiohttp-3.13.5-cp314-cp314-win32.whl", hash = "sha256:b20df693de16f42b2472a9c485e1c948ee55524786a0a34345511afdd22246f3", size = 438128 }, + { url = "https://files.pythonhosted.org/packages/22/4d/eaedff67fc805aeba4ba746aec891b4b24cebb1a7d078084b6300f79d063/aiohttp-3.13.5-cp314-cp314-win_amd64.whl", hash = "sha256:f85c6f327bf0b8c29da7d93b1cabb6363fb5e4e160a32fa241ed2dce21b73162", size = 464029 }, + { url = "https://files.pythonhosted.org/packages/79/11/c27d9332ee20d68dd164dc12a6ecdef2e2e35ecc97ed6cf0d2442844624b/aiohttp-3.13.5-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:1efb06900858bb618ff5cee184ae2de5828896c448403d51fb633f09e109be0a", size = 778758 }, + { url = "https://files.pythonhosted.org/packages/04/fb/377aead2e0a3ba5f09b7624f702a964bdf4f08b5b6728a9799830c80041e/aiohttp-3.13.5-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:fee86b7c4bd29bdaf0d53d14739b08a106fdda809ca5fe032a15f52fae5fe254", size = 512883 }, + { url = "https://files.pythonhosted.org/packages/bb/a6/aa109a33671f7a5d3bd78b46da9d852797c5e665bfda7d6b373f56bff2ec/aiohttp-3.13.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:20058e23909b9e65f9da62b396b77dfa95965cbe840f8def6e572538b1d32e36", size = 516668 }, + { url = "https://files.pythonhosted.org/packages/79/b3/ca078f9f2fa9563c36fb8ef89053ea2bb146d6f792c5104574d49d8acb63/aiohttp-3.13.5-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cf20a8d6868cb15a73cab329ffc07291ba8c22b1b88176026106ae39aa6df0f", size = 1883461 }, + { url = "https://files.pythonhosted.org/packages/b7/e3/a7ad633ca1ca497b852233a3cce6906a56c3225fb6d9217b5e5e60b7419d/aiohttp-3.13.5-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:330f5da04c987f1d5bdb8ae189137c77139f36bd1cb23779ca1a354a4b027800", size = 1747661 }, + { url = "https://files.pythonhosted.org/packages/33/b9/cd6fe579bed34a906d3d783fe60f2fa297ef55b27bb4538438ee49d4dc41/aiohttp-3.13.5-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6f1cbf0c7926d315c3c26c2da41fd2b5d2fe01ac0e157b78caefc51a782196cf", size = 1863800 }, + { url = "https://files.pythonhosted.org/packages/c0/3f/2c1e2f5144cefa889c8afd5cf431994c32f3b29da9961698ff4e3811b79a/aiohttp-3.13.5-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:53fc049ed6390d05423ba33103ded7281fe897cf97878f369a527070bd95795b", size = 1958382 }, + { url = "https://files.pythonhosted.org/packages/66/1d/f31ec3f1013723b3babe3609e7f119c2c2fb6ef33da90061a705ef3e1bc8/aiohttp-3.13.5-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:898703aa2667e3c5ca4c54ca36cd73f58b7a38ef87a5606414799ebce4d3fd3a", size = 1803724 }, + { url = "https://files.pythonhosted.org/packages/0e/b4/57712dfc6f1542f067daa81eb61da282fab3e6f1966fca25db06c4fc62d5/aiohttp-3.13.5-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0494a01ca9584eea1e5fbd6d748e61ecff218c51b576ee1999c23db7066417d8", size = 1640027 }, + { url = "https://files.pythonhosted.org/packages/25/3c/734c878fb43ec083d8e31bf029daae1beafeae582d1b35da234739e82ee7/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:6cf81fe010b8c17b09495cbd15c1d35afbc8fb405c0c9cf4738e5ae3af1d65be", size = 1806644 }, + { url = "https://files.pythonhosted.org/packages/20/a5/f671e5cbec1c21d044ff3078223f949748f3a7f86b14e34a365d74a5d21f/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:c564dd5f09ddc9d8f2c2d0a301cd30a79a2cc1b46dd1a73bef8f0038863d016b", size = 1791630 }, + { url = "https://files.pythonhosted.org/packages/0b/63/fb8d0ad63a0b8a99be97deac8c04dacf0785721c158bdf23d679a87aa99e/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2994be9f6e51046c4f864598fd9abeb4fba6e88f0b2152422c9666dcd4aea9c6", size = 1809403 }, + { url = "https://files.pythonhosted.org/packages/59/0c/bfed7f30662fcf12206481c2aac57dedee43fe1c49275e85b3a1e1742294/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:157826e2fa245d2ef46c83ea8a5faf77ca19355d278d425c29fda0beb3318037", size = 1634924 }, + { url = "https://files.pythonhosted.org/packages/17/d6/fd518d668a09fd5a3319ae5e984d4d80b9a4b3df4e21c52f02251ef5a32e/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:a8aca50daa9493e9e13c0f566201a9006f080e7c50e5e90d0b06f53146a54500", size = 1836119 }, + { url = "https://files.pythonhosted.org/packages/78/b7/15fb7a9d52e112a25b621c67b69c167805cb1f2ab8f1708a5c490d1b52fe/aiohttp-3.13.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3b13560160d07e047a93f23aaa30718606493036253d5430887514715b67c9d9", size = 1772072 }, + { url = "https://files.pythonhosted.org/packages/7e/df/57ba7f0c4a553fc2bd8b6321df236870ec6fd64a2a473a8a13d4f733214e/aiohttp-3.13.5-cp314-cp314t-win32.whl", hash = "sha256:9a0f4474b6ea6818b41f82172d799e4b3d29e22c2c520ce4357856fced9af2f8", size = 471819 }, + { url = "https://files.pythonhosted.org/packages/62/29/2f8418269e46454a26171bfdd6a055d74febf32234e474930f2f60a17145/aiohttp-3.13.5-cp314-cp314t-win_amd64.whl", hash = "sha256:18a2f6c1182c51baa1d28d68fea51513cb2a76612f038853c0ad3c145423d3d9", size = 505441 }, +] + +[[package]] +name = "aiosignal" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "frozenlist" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490 }, +] + +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303 }, +] + +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643 }, +] + +[[package]] +name = "anyio" +version = "4.13.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/2c5dd9f512b66549ae92767a9c7b330ae88e1932ca57876909410251fe13/anyio-4.13.0.tar.gz", hash = "sha256:334b70e641fd2221c1505b3890c69882fe4a2df910cba14d97019b90b24439dc", size = 231622 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353 }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548 }, +] + +[[package]] +name = "backoff" +version = "2.2.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/d7/5bbeb12c44d7c4f2fb5b56abce497eb5ed9f34d85701de869acedd602619/backoff-2.2.1.tar.gz", hash = "sha256:03f829f5bb1923180821643f8753b0502c3b682293992485b0eef2807afa5cba", size = 17001 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/73/b6e24bd22e6720ca8ee9a85a0c4a2971af8497d8f3193fa05390cbd46e09/backoff-2.2.1-py3-none-any.whl", hash = "sha256:63579f9a0628e06278f7e47b7d7d5b6ce20dc65c5e96a6f3ca99a6adca0396e8", size = 15148 }, +] + +[[package]] +name = "boto3" +version = "1.43.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/4b/616367e871ce3f1cb3e8545a97736b6331b9fb081497f2d44c5b2aa6959d/boto3-1.43.14.tar.gz", hash = "sha256:5c0a994b3182061ee101812e721100717a4d664f9f4ceaf4a86b6d032ce9fc2d", size = 113142 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/00/59cb9329c18e2d3aa23062ceaa87d065f2e81e7d2931df24d64e9a7815aa/boto3-1.43.14-py3-none-any.whl", hash = "sha256:574335744656cfed0b362a0a0467aaf2eb2bf15526edcd02d31d3c661f4b09e4", size = 140536 }, +] + +[[package]] +name = "botocore" +version = "1.43.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/3c/798d2f7deb118241930c7c6bcfb0b970d3f0245bf580700663199aeed2c3/botocore-1.43.14.tar.gz", hash = "sha256:b9e500737e43d2f147c9d4e23b54360335e77d4c0ba90a318f51b65e06cb8516", size = 15382604 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/7e/6e64821077cd2efc4aa51b7d638fb6d48e1c7c450201c529fbaf1de8bfd3/botocore-1.43.14-py3-none-any.whl", hash = "sha256:1f4a2a95ea78c10398e78431e98c1fe47adb54a7b10a32975144c1f541186658", size = 15061424 }, +] + +[[package]] +name = "certifi" +version = "2026.5.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/ce/ee2ecad540810a79593028e88299baeae54d346cc7a0d94b6199988b89b1/certifi-2026.5.20.tar.gz", hash = "sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d", size = 135422 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/8c/57e832b7af6d7c5abe66eb3fbe3a3a32f4d11ea23a1aa7131371035be991/certifi-2026.5.20-py3-none-any.whl", hash = "sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897", size = 134134 }, +] + +[[package]] +name = "cffi" +version = "2.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/4a/3dfd5f7850cbf0d06dc84ba9aa00db766b52ca38d8b86e3a38314d52498c/cffi-2.0.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:b4c854ef3adc177950a8dfc81a86f5115d2abd545751a304c5bcf2c2c7283cfe", size = 184344 }, + { url = "https://files.pythonhosted.org/packages/4f/8b/f0e4c441227ba756aafbe78f117485b25bb26b1c059d01f137fa6d14896b/cffi-2.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2de9a304e27f7596cd03d16f1b7c72219bd944e99cc52b84d0145aefb07cbd3c", size = 180560 }, + { url = "https://files.pythonhosted.org/packages/b1/b7/1200d354378ef52ec227395d95c2576330fd22a869f7a70e88e1447eb234/cffi-2.0.0-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:baf5215e0ab74c16e2dd324e8ec067ef59e41125d3eade2b863d294fd5035c92", size = 209613 }, + { url = "https://files.pythonhosted.org/packages/b8/56/6033f5e86e8cc9bb629f0077ba71679508bdf54a9a5e112a3c0b91870332/cffi-2.0.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:730cacb21e1bdff3ce90babf007d0a0917cc3e6492f336c2f0134101e0944f93", size = 216476 }, + { url = "https://files.pythonhosted.org/packages/dc/7f/55fecd70f7ece178db2f26128ec41430d8720f2d12ca97bf8f0a628207d5/cffi-2.0.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6824f87845e3396029f3820c206e459ccc91760e8fa24422f8b0c3d1731cbec5", size = 203374 }, + { url = "https://files.pythonhosted.org/packages/84/ef/a7b77c8bdc0f77adc3b46888f1ad54be8f3b7821697a7b89126e829e676a/cffi-2.0.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:9de40a7b0323d889cf8d23d1ef214f565ab154443c42737dfe52ff82cf857664", size = 202597 }, + { url = "https://files.pythonhosted.org/packages/d7/91/500d892b2bf36529a75b77958edfcd5ad8e2ce4064ce2ecfeab2125d72d1/cffi-2.0.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8941aaadaf67246224cee8c3803777eed332a19d909b47e29c9842ef1e79ac26", size = 215574 }, + { url = "https://files.pythonhosted.org/packages/44/64/58f6255b62b101093d5df22dcb752596066c7e89dd725e0afaed242a61be/cffi-2.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:a05d0c237b3349096d3981b727493e22147f934b20f6f125a3eba8f994bec4a9", size = 218971 }, + { url = "https://files.pythonhosted.org/packages/ab/49/fa72cebe2fd8a55fbe14956f9970fe8eb1ac59e5df042f603ef7c8ba0adc/cffi-2.0.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:94698a9c5f91f9d138526b48fe26a199609544591f859c870d477351dc7b2414", size = 211972 }, + { url = "https://files.pythonhosted.org/packages/0b/28/dd0967a76aab36731b6ebfe64dec4e981aff7e0608f60c2d46b46982607d/cffi-2.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:5fed36fccc0612a53f1d4d9a816b50a36702c28a2aa880cb8a122b3466638743", size = 217078 }, + { url = "https://files.pythonhosted.org/packages/2b/c0/015b25184413d7ab0a410775fdb4a50fca20f5589b5dab1dbbfa3baad8ce/cffi-2.0.0-cp311-cp311-win32.whl", hash = "sha256:c649e3a33450ec82378822b3dad03cc228b8f5963c0c12fc3b1e0ab940f768a5", size = 172076 }, + { url = "https://files.pythonhosted.org/packages/ae/8f/dc5531155e7070361eb1b7e4c1a9d896d0cb21c49f807a6c03fd63fc877e/cffi-2.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:66f011380d0e49ed280c789fbd08ff0d40968ee7b665575489afa95c98196ab5", size = 182820 }, + { url = "https://files.pythonhosted.org/packages/95/5c/1b493356429f9aecfd56bc171285a4c4ac8697f76e9bbbbb105e537853a1/cffi-2.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:c6638687455baf640e37344fe26d37c404db8b80d037c3d29f58fe8d1c3b194d", size = 177635 }, + { url = "https://files.pythonhosted.org/packages/ea/47/4f61023ea636104d4f16ab488e268b93008c3d0bb76893b1b31db1f96802/cffi-2.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d02d6655b0e54f54c4ef0b94eb6be0607b70853c45ce98bd278dc7de718be5d", size = 185271 }, + { url = "https://files.pythonhosted.org/packages/df/a2/781b623f57358e360d62cdd7a8c681f074a71d445418a776eef0aadb4ab4/cffi-2.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8eca2a813c1cb7ad4fb74d368c2ffbbb4789d377ee5bb8df98373c2cc0dee76c", size = 181048 }, + { url = "https://files.pythonhosted.org/packages/ff/df/a4f0fbd47331ceeba3d37c2e51e9dfc9722498becbeec2bd8bc856c9538a/cffi-2.0.0-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:21d1152871b019407d8ac3985f6775c079416c282e431a4da6afe7aefd2bccbe", size = 212529 }, + { url = "https://files.pythonhosted.org/packages/d5/72/12b5f8d3865bf0f87cf1404d8c374e7487dcf097a1c91c436e72e6badd83/cffi-2.0.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b21e08af67b8a103c71a250401c78d5e0893beff75e28c53c98f4de42f774062", size = 220097 }, + { url = "https://files.pythonhosted.org/packages/c2/95/7a135d52a50dfa7c882ab0ac17e8dc11cec9d55d2c18dda414c051c5e69e/cffi-2.0.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:1e3a615586f05fc4065a8b22b8152f0c1b00cdbc60596d187c2a74f9e3036e4e", size = 207983 }, + { url = "https://files.pythonhosted.org/packages/3a/c8/15cb9ada8895957ea171c62dc78ff3e99159ee7adb13c0123c001a2546c1/cffi-2.0.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:81afed14892743bbe14dacb9e36d9e0e504cd204e0b165062c488942b9718037", size = 206519 }, + { url = "https://files.pythonhosted.org/packages/78/2d/7fa73dfa841b5ac06c7b8855cfc18622132e365f5b81d02230333ff26e9e/cffi-2.0.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3e17ed538242334bf70832644a32a7aae3d83b57567f9fd60a26257e992b79ba", size = 219572 }, + { url = "https://files.pythonhosted.org/packages/07/e0/267e57e387b4ca276b90f0434ff88b2c2241ad72b16d31836adddfd6031b/cffi-2.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3925dd22fa2b7699ed2617149842d2e6adde22b262fcbfada50e3d195e4b3a94", size = 222963 }, + { url = "https://files.pythonhosted.org/packages/b6/75/1f2747525e06f53efbd878f4d03bac5b859cbc11c633d0fb81432d98a795/cffi-2.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2c8f814d84194c9ea681642fd164267891702542f028a15fc97d4674b6206187", size = 221361 }, + { url = "https://files.pythonhosted.org/packages/7b/2b/2b6435f76bfeb6bbf055596976da087377ede68df465419d192acf00c437/cffi-2.0.0-cp312-cp312-win32.whl", hash = "sha256:da902562c3e9c550df360bfa53c035b2f241fed6d9aef119048073680ace4a18", size = 172932 }, + { url = "https://files.pythonhosted.org/packages/f8/ed/13bd4418627013bec4ed6e54283b1959cf6db888048c7cf4b4c3b5b36002/cffi-2.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:da68248800ad6320861f129cd9c1bf96ca849a2771a59e0344e88681905916f5", size = 183557 }, + { url = "https://files.pythonhosted.org/packages/95/31/9f7f93ad2f8eff1dbc1c3656d7ca5bfd8fb52c9d786b4dcf19b2d02217fa/cffi-2.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:4671d9dd5ec934cb9a73e7ee9676f9362aba54f7f34910956b84d727b0d73fb6", size = 177762 }, + { url = "https://files.pythonhosted.org/packages/4b/8d/a0a47a0c9e413a658623d014e91e74a50cdd2c423f7ccfd44086ef767f90/cffi-2.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:00bdf7acc5f795150faa6957054fbbca2439db2f775ce831222b66f192f03beb", size = 185230 }, + { url = "https://files.pythonhosted.org/packages/4a/d2/a6c0296814556c68ee32009d9c2ad4f85f2707cdecfd7727951ec228005d/cffi-2.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:45d5e886156860dc35862657e1494b9bae8dfa63bf56796f2fb56e1679fc0bca", size = 181043 }, + { url = "https://files.pythonhosted.org/packages/b0/1e/d22cc63332bd59b06481ceaac49d6c507598642e2230f201649058a7e704/cffi-2.0.0-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:07b271772c100085dd28b74fa0cd81c8fb1a3ba18b21e03d7c27f3436a10606b", size = 212446 }, + { url = "https://files.pythonhosted.org/packages/a9/f5/a2c23eb03b61a0b8747f211eb716446c826ad66818ddc7810cc2cc19b3f2/cffi-2.0.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d48a880098c96020b02d5a1f7d9251308510ce8858940e6fa99ece33f610838b", size = 220101 }, + { url = "https://files.pythonhosted.org/packages/f2/7f/e6647792fc5850d634695bc0e6ab4111ae88e89981d35ac269956605feba/cffi-2.0.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f93fd8e5c8c0a4aa1f424d6173f14a892044054871c771f8566e4008eaa359d2", size = 207948 }, + { url = "https://files.pythonhosted.org/packages/cb/1e/a5a1bd6f1fb30f22573f76533de12a00bf274abcdc55c8edab639078abb6/cffi-2.0.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:dd4f05f54a52fb558f1ba9f528228066954fee3ebe629fc1660d874d040ae5a3", size = 206422 }, + { url = "https://files.pythonhosted.org/packages/98/df/0a1755e750013a2081e863e7cd37e0cdd02664372c754e5560099eb7aa44/cffi-2.0.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c8d3b5532fc71b7a77c09192b4a5a200ea992702734a2e9279a37f2478236f26", size = 219499 }, + { url = "https://files.pythonhosted.org/packages/50/e1/a969e687fcf9ea58e6e2a928ad5e2dd88cc12f6f0ab477e9971f2309b57c/cffi-2.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9b29c1f0ae438d5ee9acb31cadee00a58c46cc9c0b2f9038c6b0b3470877a8c", size = 222928 }, + { url = "https://files.pythonhosted.org/packages/36/54/0362578dd2c9e557a28ac77698ed67323ed5b9775ca9d3fe73fe191bb5d8/cffi-2.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:6d50360be4546678fc1b79ffe7a66265e28667840010348dd69a314145807a1b", size = 221302 }, + { url = "https://files.pythonhosted.org/packages/eb/6d/bf9bda840d5f1dfdbf0feca87fbdb64a918a69bca42cfa0ba7b137c48cb8/cffi-2.0.0-cp313-cp313-win32.whl", hash = "sha256:74a03b9698e198d47562765773b4a8309919089150a0bb17d829ad7b44b60d27", size = 172909 }, + { url = "https://files.pythonhosted.org/packages/37/18/6519e1ee6f5a1e579e04b9ddb6f1676c17368a7aba48299c3759bbc3c8b3/cffi-2.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:19f705ada2530c1167abacb171925dd886168931e0a7b78f5bffcae5c6b5be75", size = 183402 }, + { url = "https://files.pythonhosted.org/packages/cb/0e/02ceeec9a7d6ee63bb596121c2c8e9b3a9e150936f4fbef6ca1943e6137c/cffi-2.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:256f80b80ca3853f90c21b23ee78cd008713787b1b1e93eae9f3d6a7134abd91", size = 177780 }, + { url = "https://files.pythonhosted.org/packages/92/c4/3ce07396253a83250ee98564f8d7e9789fab8e58858f35d07a9a2c78de9f/cffi-2.0.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fc33c5141b55ed366cfaad382df24fe7dcbc686de5be719b207bb248e3053dc5", size = 185320 }, + { url = "https://files.pythonhosted.org/packages/59/dd/27e9fa567a23931c838c6b02d0764611c62290062a6d4e8ff7863daf9730/cffi-2.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c654de545946e0db659b3400168c9ad31b5d29593291482c43e3564effbcee13", size = 181487 }, + { url = "https://files.pythonhosted.org/packages/d6/43/0e822876f87ea8a4ef95442c3d766a06a51fc5298823f884ef87aaad168c/cffi-2.0.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:24b6f81f1983e6df8db3adc38562c83f7d4a0c36162885ec7f7b77c7dcbec97b", size = 220049 }, + { url = "https://files.pythonhosted.org/packages/b4/89/76799151d9c2d2d1ead63c2429da9ea9d7aac304603de0c6e8764e6e8e70/cffi-2.0.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:12873ca6cb9b0f0d3a0da705d6086fe911591737a59f28b7936bdfed27c0d47c", size = 207793 }, + { url = "https://files.pythonhosted.org/packages/bb/dd/3465b14bb9e24ee24cb88c9e3730f6de63111fffe513492bf8c808a3547e/cffi-2.0.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:d9b97165e8aed9272a6bb17c01e3cc5871a594a446ebedc996e2397a1c1ea8ef", size = 206300 }, + { url = "https://files.pythonhosted.org/packages/47/d9/d83e293854571c877a92da46fdec39158f8d7e68da75bf73581225d28e90/cffi-2.0.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:afb8db5439b81cf9c9d0c80404b60c3cc9c3add93e114dcae767f1477cb53775", size = 219244 }, + { url = "https://files.pythonhosted.org/packages/2b/0f/1f177e3683aead2bb00f7679a16451d302c436b5cbf2505f0ea8146ef59e/cffi-2.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:737fe7d37e1a1bffe70bd5754ea763a62a066dc5913ca57e957824b72a85e205", size = 222828 }, + { url = "https://files.pythonhosted.org/packages/c6/0f/cafacebd4b040e3119dcb32fed8bdef8dfe94da653155f9d0b9dc660166e/cffi-2.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:38100abb9d1b1435bc4cc340bb4489635dc2f0da7456590877030c9b3d40b0c1", size = 220926 }, + { url = "https://files.pythonhosted.org/packages/3e/aa/df335faa45b395396fcbc03de2dfcab242cd61a9900e914fe682a59170b1/cffi-2.0.0-cp314-cp314-win32.whl", hash = "sha256:087067fa8953339c723661eda6b54bc98c5625757ea62e95eb4898ad5e776e9f", size = 175328 }, + { url = "https://files.pythonhosted.org/packages/bb/92/882c2d30831744296ce713f0feb4c1cd30f346ef747b530b5318715cc367/cffi-2.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:203a48d1fb583fc7d78a4c6655692963b860a417c0528492a6bc21f1aaefab25", size = 185650 }, + { url = "https://files.pythonhosted.org/packages/9f/2c/98ece204b9d35a7366b5b2c6539c350313ca13932143e79dc133ba757104/cffi-2.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:dbd5c7a25a7cb98f5ca55d258b103a2054f859a46ae11aaf23134f9cc0d356ad", size = 180687 }, + { url = "https://files.pythonhosted.org/packages/3e/61/c768e4d548bfa607abcda77423448df8c471f25dbe64fb2ef6d555eae006/cffi-2.0.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:9a67fc9e8eb39039280526379fb3a70023d77caec1852002b4da7e8b270c4dd9", size = 188773 }, + { url = "https://files.pythonhosted.org/packages/2c/ea/5f76bce7cf6fcd0ab1a1058b5af899bfbef198bea4d5686da88471ea0336/cffi-2.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7a66c7204d8869299919db4d5069a82f1561581af12b11b3c9f48c584eb8743d", size = 185013 }, + { url = "https://files.pythonhosted.org/packages/be/b4/c56878d0d1755cf9caa54ba71e5d049479c52f9e4afc230f06822162ab2f/cffi-2.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7cc09976e8b56f8cebd752f7113ad07752461f48a58cbba644139015ac24954c", size = 221593 }, + { url = "https://files.pythonhosted.org/packages/e0/0d/eb704606dfe8033e7128df5e90fee946bbcb64a04fcdaa97321309004000/cffi-2.0.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:92b68146a71df78564e4ef48af17551a5ddd142e5190cdf2c5624d0c3ff5b2e8", size = 209354 }, + { url = "https://files.pythonhosted.org/packages/d8/19/3c435d727b368ca475fb8742ab97c9cb13a0de600ce86f62eab7fa3eea60/cffi-2.0.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:b1e74d11748e7e98e2f426ab176d4ed720a64412b6a15054378afdb71e0f37dc", size = 208480 }, + { url = "https://files.pythonhosted.org/packages/d0/44/681604464ed9541673e486521497406fadcc15b5217c3e326b061696899a/cffi-2.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:28a3a209b96630bca57cce802da70c266eb08c6e97e5afd61a75611ee6c64592", size = 221584 }, + { url = "https://files.pythonhosted.org/packages/25/8e/342a504ff018a2825d395d44d63a767dd8ebc927ebda557fecdaca3ac33a/cffi-2.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7553fb2090d71822f02c629afe6042c299edf91ba1bf94951165613553984512", size = 224443 }, + { url = "https://files.pythonhosted.org/packages/e1/5e/b666bacbbc60fbf415ba9988324a132c9a7a0448a9a8f125074671c0f2c3/cffi-2.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6c6c373cfc5c83a975506110d17457138c8c63016b563cc9ed6e056a82f13ce4", size = 223437 }, + { url = "https://files.pythonhosted.org/packages/a0/1d/ec1a60bd1a10daa292d3cd6bb0b359a81607154fb8165f3ec95fe003b85c/cffi-2.0.0-cp314-cp314t-win32.whl", hash = "sha256:1fc9ea04857caf665289b7a75923f2c6ed559b8298a1b8c49e59f7dd95c8481e", size = 180487 }, + { url = "https://files.pythonhosted.org/packages/bf/41/4c1168c74fac325c0c8156f04b6749c8b6a8f405bbf91413ba088359f60d/cffi-2.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d68b6cef7827e8641e8ef16f4494edda8b36104d79773a334beaa1e3521430f6", size = 191726 }, + { url = "https://files.pythonhosted.org/packages/ae/3a/dbeec9d1ee0844c679f6bb5d6ad4e9f198b1224f4e7a32825f47f6192b0c/cffi-2.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0a1527a803f0a659de1af2e1fd700213caba79377e27e4693648c2923da066f9", size = 184195 }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/a1/67fe25fac3c7642725500a3f6cfe5821ad557c3abb11c9d20d12c7008d3e/charset_normalizer-3.4.7.tar.gz", hash = "sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5", size = 144271 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/d7/b5b7020a0565c2e9fa8c09f4b5fa6232feb326b8c20081ccded47ea368fd/charset_normalizer-3.4.7-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7", size = 309705 }, + { url = "https://files.pythonhosted.org/packages/5a/53/58c29116c340e5456724ecd2fff4196d236b98f3da97b404bc5e51ac3493/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7", size = 206419 }, + { url = "https://files.pythonhosted.org/packages/b2/02/e8146dc6591a37a00e5144c63f29fb7c97a734ea8a111190783c0e60ab63/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e", size = 227901 }, + { url = "https://files.pythonhosted.org/packages/fb/73/77486c4cd58f1267bf17db420e930c9afa1b3be3fe8c8b8ebbebc9624359/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c", size = 222742 }, + { url = "https://files.pythonhosted.org/packages/a1/fa/f74eb381a7d94ded44739e9d94de18dc5edc9c17fb8c11f0a6890696c0a9/charset_normalizer-3.4.7-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df", size = 214061 }, + { url = "https://files.pythonhosted.org/packages/dc/92/42bd3cefcf7687253fb86694b45f37b733c97f59af3724f356fa92b8c344/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265", size = 199239 }, + { url = "https://files.pythonhosted.org/packages/4c/3d/069e7184e2aa3b3cddc700e3dd267413dc259854adc3380421c805c6a17d/charset_normalizer-3.4.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4", size = 210173 }, + { url = "https://files.pythonhosted.org/packages/62/51/9d56feb5f2e7074c46f93e0ebdbe61f0848ee246e2f0d89f8e20b89ebb8f/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e", size = 209841 }, + { url = "https://files.pythonhosted.org/packages/d2/59/893d8f99cc4c837dda1fe2f1139079703deb9f321aabcb032355de13b6c7/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38", size = 200304 }, + { url = "https://files.pythonhosted.org/packages/7d/1d/ee6f3be3464247578d1ed5c46de545ccc3d3ff933695395c402c21fa6b77/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c", size = 229455 }, + { url = "https://files.pythonhosted.org/packages/54/bb/8fb0a946296ea96a488928bdce8ef99023998c48e4713af533e9bb98ef07/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b", size = 210036 }, + { url = "https://files.pythonhosted.org/packages/9a/bc/015b2387f913749f82afd4fcba07846d05b6d784dd16123cb66860e0237d/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c", size = 224739 }, + { url = "https://files.pythonhosted.org/packages/17/ab/63133691f56baae417493cba6b7c641571a2130eb7bceba6773367ab9ec5/charset_normalizer-3.4.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d", size = 216277 }, + { url = "https://files.pythonhosted.org/packages/06/6d/3be70e827977f20db77c12a97e6a9f973631a45b8d186c084527e53e77a4/charset_normalizer-3.4.7-cp311-cp311-win32.whl", hash = "sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad", size = 147819 }, + { url = "https://files.pythonhosted.org/packages/20/d9/5f67790f06b735d7c7637171bbfd89882ad67201891b7275e51116ed8207/charset_normalizer-3.4.7-cp311-cp311-win_amd64.whl", hash = "sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00", size = 159281 }, + { url = "https://files.pythonhosted.org/packages/ca/83/6413f36c5a34afead88ce6f66684d943d91f233d76dd083798f9602b75ae/charset_normalizer-3.4.7-cp311-cp311-win_arm64.whl", hash = "sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1", size = 147843 }, + { url = "https://files.pythonhosted.org/packages/0c/eb/4fc8d0a7110eb5fc9cc161723a34a8a6c200ce3b4fbf681bc86feee22308/charset_normalizer-3.4.7-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46", size = 311328 }, + { url = "https://files.pythonhosted.org/packages/f8/e3/0fadc706008ac9d7b9b5be6dc767c05f9d3e5df51744ce4cc9605de7b9f4/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2", size = 208061 }, + { url = "https://files.pythonhosted.org/packages/42/f0/3dd1045c47f4a4604df85ec18ad093912ae1344ac706993aff91d38773a2/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b", size = 229031 }, + { url = "https://files.pythonhosted.org/packages/dc/67/675a46eb016118a2fbde5a277a5d15f4f69d5f3f5f338e5ee2f8948fcf43/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a", size = 225239 }, + { url = "https://files.pythonhosted.org/packages/4b/f8/d0118a2f5f23b02cd166fa385c60f9b0d4f9194f574e2b31cef350ad7223/charset_normalizer-3.4.7-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116", size = 216589 }, + { url = "https://files.pythonhosted.org/packages/b1/f1/6d2b0b261b6c4ceef0fcb0d17a01cc5bc53586c2d4796fa04b5c540bc13d/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb", size = 202733 }, + { url = "https://files.pythonhosted.org/packages/6f/c0/7b1f943f7e87cc3db9626ba17807d042c38645f0a1d4415c7a14afb5591f/charset_normalizer-3.4.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1", size = 212652 }, + { url = "https://files.pythonhosted.org/packages/38/dd/5a9ab159fe45c6e72079398f277b7d2b523e7f716acc489726115a910097/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15", size = 211229 }, + { url = "https://files.pythonhosted.org/packages/d5/ff/531a1cad5ca855d1c1a8b69cb71abfd6d85c0291580146fda7c82857caa1/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5", size = 203552 }, + { url = "https://files.pythonhosted.org/packages/c1/4c/a5fb52d528a8ca41f7598cb619409ece30a169fbdf9cdce592e53b46c3a6/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d", size = 230806 }, + { url = "https://files.pythonhosted.org/packages/59/7a/071feed8124111a32b316b33ae4de83d36923039ef8cf48120266844285b/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7", size = 212316 }, + { url = "https://files.pythonhosted.org/packages/fd/35/f7dba3994312d7ba508e041eaac39a36b120f32d4c8662b8814dab876431/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464", size = 227274 }, + { url = "https://files.pythonhosted.org/packages/8a/2d/a572df5c9204ab7688ec1edc895a73ebded3b023bb07364710b05dd1c9be/charset_normalizer-3.4.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49", size = 218468 }, + { url = "https://files.pythonhosted.org/packages/86/eb/890922a8b03a568ca2f336c36585a4713c55d4d67bf0f0c78924be6315ca/charset_normalizer-3.4.7-cp312-cp312-win32.whl", hash = "sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c", size = 148460 }, + { url = "https://files.pythonhosted.org/packages/35/d9/0e7dffa06c5ab081f75b1b786f0aefc88365825dfcd0ac544bdb7b2b6853/charset_normalizer-3.4.7-cp312-cp312-win_amd64.whl", hash = "sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6", size = 159330 }, + { url = "https://files.pythonhosted.org/packages/9e/5d/481bcc2a7c88ea6b0878c299547843b2521ccbc40980cb406267088bc701/charset_normalizer-3.4.7-cp312-cp312-win_arm64.whl", hash = "sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d", size = 147828 }, + { url = "https://files.pythonhosted.org/packages/c1/3b/66777e39d3ae1ddc77ee606be4ec6d8cbd4c801f65e5a1b6f2b11b8346dd/charset_normalizer-3.4.7-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063", size = 309627 }, + { url = "https://files.pythonhosted.org/packages/2e/4e/b7f84e617b4854ade48a1b7915c8ccfadeba444d2a18c291f696e37f0d3b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c", size = 207008 }, + { url = "https://files.pythonhosted.org/packages/c4/bb/ec73c0257c9e11b268f018f068f5d00aa0ef8c8b09f7753ebd5f2880e248/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66", size = 228303 }, + { url = "https://files.pythonhosted.org/packages/85/fb/32d1f5033484494619f701e719429c69b766bfc4dbc61aa9e9c8c166528b/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18", size = 224282 }, + { url = "https://files.pythonhosted.org/packages/fa/07/330e3a0dda4c404d6da83b327270906e9654a24f6c546dc886a0eb0ffb23/charset_normalizer-3.4.7-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd", size = 215595 }, + { url = "https://files.pythonhosted.org/packages/e3/7c/fc890655786e423f02556e0216d4b8c6bcb6bdfa890160dc66bf52dee468/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215", size = 201986 }, + { url = "https://files.pythonhosted.org/packages/d8/97/bfb18b3db2aed3b90cf54dc292ad79fdd5ad65c4eae454099475cbeadd0d/charset_normalizer-3.4.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859", size = 211711 }, + { url = "https://files.pythonhosted.org/packages/6f/a5/a581c13798546a7fd557c82614a5c65a13df2157e9ad6373166d2a3e645d/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8", size = 210036 }, + { url = "https://files.pythonhosted.org/packages/8c/bf/b3ab5bcb478e4193d517644b0fb2bf5497fbceeaa7a1bc0f4d5b50953861/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5", size = 202998 }, + { url = "https://files.pythonhosted.org/packages/e7/4e/23efd79b65d314fa320ec6017b4b5834d5c12a58ba4610aa353af2e2f577/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832", size = 230056 }, + { url = "https://files.pythonhosted.org/packages/b9/9f/1e1941bc3f0e01df116e68dc37a55c4d249df5e6fa77f008841aef68264f/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6", size = 211537 }, + { url = "https://files.pythonhosted.org/packages/80/0f/088cbb3020d44428964a6c97fe1edfb1b9550396bf6d278330281e8b709c/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48", size = 226176 }, + { url = "https://files.pythonhosted.org/packages/6a/9f/130394f9bbe06f4f63e22641d32fc9b202b7e251c9aef4db044324dac493/charset_normalizer-3.4.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a", size = 217723 }, + { url = "https://files.pythonhosted.org/packages/73/55/c469897448a06e49f8fa03f6caae97074fde823f432a98f979cc42b90e69/charset_normalizer-3.4.7-cp313-cp313-win32.whl", hash = "sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e", size = 148085 }, + { url = "https://files.pythonhosted.org/packages/5d/78/1b74c5bbb3f99b77a1715c91b3e0b5bdb6fe302d95ace4f5b1bec37b0167/charset_normalizer-3.4.7-cp313-cp313-win_amd64.whl", hash = "sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110", size = 158819 }, + { url = "https://files.pythonhosted.org/packages/68/86/46bd42279d323deb8687c4a5a811fd548cb7d1de10cf6535d099877a9a9f/charset_normalizer-3.4.7-cp313-cp313-win_arm64.whl", hash = "sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b", size = 147915 }, + { url = "https://files.pythonhosted.org/packages/97/c8/c67cb8c70e19ef1960b97b22ed2a1567711de46c4ddf19799923adc836c2/charset_normalizer-3.4.7-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0", size = 309234 }, + { url = "https://files.pythonhosted.org/packages/99/85/c091fdee33f20de70d6c8b522743b6f831a2f1cd3ff86de4c6a827c48a76/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a", size = 208042 }, + { url = "https://files.pythonhosted.org/packages/87/1c/ab2ce611b984d2fd5d86a5a8a19c1ae26acac6bad967da4967562c75114d/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b", size = 228706 }, + { url = "https://files.pythonhosted.org/packages/a8/29/2b1d2cb00bf085f59d29eb773ce58ec2d325430f8c216804a0a5cd83cbca/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41", size = 224727 }, + { url = "https://files.pythonhosted.org/packages/47/5c/032c2d5a07fe4d4855fea851209cca2b6f03ebeb6d4e3afdb3358386a684/charset_normalizer-3.4.7-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e", size = 215882 }, + { url = "https://files.pythonhosted.org/packages/2c/c2/356065d5a8b78ed04499cae5f339f091946a6a74f91e03476c33f0ab7100/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae", size = 200860 }, + { url = "https://files.pythonhosted.org/packages/0c/cd/a32a84217ced5039f53b29f460962abb2d4420def55afabe45b1c3c7483d/charset_normalizer-3.4.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18", size = 211564 }, + { url = "https://files.pythonhosted.org/packages/44/86/58e6f13ce26cc3b8f4a36b94a0f22ae2f00a72534520f4ae6857c4b81f89/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b", size = 211276 }, + { url = "https://files.pythonhosted.org/packages/8f/fe/d17c32dc72e17e155e06883efa84514ca375f8a528ba2546bee73fc4df81/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356", size = 201238 }, + { url = "https://files.pythonhosted.org/packages/6a/29/f33daa50b06525a237451cdb6c69da366c381a3dadcd833fa5676bc468b3/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab", size = 230189 }, + { url = "https://files.pythonhosted.org/packages/b6/6e/52c84015394a6a0bdcd435210a7e944c5f94ea1055f5cc5d56c5fe368e7b/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46", size = 211352 }, + { url = "https://files.pythonhosted.org/packages/8c/d7/4353be581b373033fb9198bf1da3cf8f09c1082561e8e922aa7b39bf9fe8/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44", size = 227024 }, + { url = "https://files.pythonhosted.org/packages/30/45/99d18aa925bd1740098ccd3060e238e21115fffbfdcb8f3ece837d0ace6c/charset_normalizer-3.4.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72", size = 217869 }, + { url = "https://files.pythonhosted.org/packages/5c/05/5ee478aa53f4bb7996482153d4bfe1b89e0f087f0ab6b294fcf92d595873/charset_normalizer-3.4.7-cp314-cp314-win32.whl", hash = "sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10", size = 148541 }, + { url = "https://files.pythonhosted.org/packages/48/77/72dcb0921b2ce86420b2d79d454c7022bf5be40202a2a07906b9f2a35c97/charset_normalizer-3.4.7-cp314-cp314-win_amd64.whl", hash = "sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f", size = 159634 }, + { url = "https://files.pythonhosted.org/packages/c6/a3/c2369911cd72f02386e4e340770f6e158c7980267da16af8f668217abaa0/charset_normalizer-3.4.7-cp314-cp314-win_arm64.whl", hash = "sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246", size = 148384 }, + { url = "https://files.pythonhosted.org/packages/94/09/7e8a7f73d24dba1f0035fbbf014d2c36828fc1bf9c88f84093e57d315935/charset_normalizer-3.4.7-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24", size = 330133 }, + { url = "https://files.pythonhosted.org/packages/8d/da/96975ddb11f8e977f706f45cddd8540fd8242f71ecdb5d18a80723dcf62c/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79", size = 216257 }, + { url = "https://files.pythonhosted.org/packages/e5/e8/1d63bf8ef2d388e95c64b2098f45f84758f6d102a087552da1485912637b/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960", size = 234851 }, + { url = "https://files.pythonhosted.org/packages/9b/40/e5ff04233e70da2681fa43969ad6f66ca5611d7e669be0246c4c7aaf6dc8/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4", size = 233393 }, + { url = "https://files.pythonhosted.org/packages/be/c1/06c6c49d5a5450f76899992f1ee40b41d076aee9279b49cf9974d2f313d5/charset_normalizer-3.4.7-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e", size = 223251 }, + { url = "https://files.pythonhosted.org/packages/2b/9f/f2ff16fb050946169e3e1f82134d107e5d4ae72647ec8a1b1446c148480f/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1", size = 206609 }, + { url = "https://files.pythonhosted.org/packages/69/d5/a527c0cd8d64d2eab7459784fb4169a0ac76e5a6fc5237337982fd61347e/charset_normalizer-3.4.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44", size = 220014 }, + { url = "https://files.pythonhosted.org/packages/7e/80/8a7b8104a3e203074dc9aa2c613d4b726c0e136bad1cc734594b02867972/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e", size = 218979 }, + { url = "https://files.pythonhosted.org/packages/02/9a/b759b503d507f375b2b5c153e4d2ee0a75aa215b7f2489cf314f4541f2c0/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3", size = 209238 }, + { url = "https://files.pythonhosted.org/packages/c2/4e/0f3f5d47b86bdb79256e7290b26ac847a2832d9a4033f7eb2cd4bcf4bb5b/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0", size = 236110 }, + { url = "https://files.pythonhosted.org/packages/96/23/bce28734eb3ed2c91dcf93abeb8a5cf393a7b2749725030bb630e554fdd8/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e", size = 219824 }, + { url = "https://files.pythonhosted.org/packages/2c/6f/6e897c6984cc4d41af319b077f2f600fc8214eb2fe2d6bcb79141b882400/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb", size = 233103 }, + { url = "https://files.pythonhosted.org/packages/76/22/ef7bd0fe480a0ae9b656189ec00744b60933f68b4f42a7bb06589f6f576a/charset_normalizer-3.4.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe", size = 225194 }, + { url = "https://files.pythonhosted.org/packages/c5/a7/0e0ab3e0b5bc1219bd80a6a0d4d72ca74d9250cb2382b7c699c147e06017/charset_normalizer-3.4.7-cp314-cp314t-win32.whl", hash = "sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0", size = 159827 }, + { url = "https://files.pythonhosted.org/packages/7a/1d/29d32e0fb40864b1f878c7f5a0b343ae676c6e2b271a2d55cc3a152391da/charset_normalizer-3.4.7-cp314-cp314t-win_amd64.whl", hash = "sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c", size = 174168 }, + { url = "https://files.pythonhosted.org/packages/de/32/d92444ad05c7a6e41fb2036749777c163baf7a0301a040cb672d6b2b1ae9/charset_normalizer-3.4.7-cp314-cp314t-win_arm64.whl", hash = "sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d", size = 153018 }, + { url = "https://files.pythonhosted.org/packages/db/8f/61959034484a4a7c527811f4721e75d02d653a35afb0b6054474d8185d4c/charset_normalizer-3.4.7-py3-none-any.whl", hash = "sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d", size = 61958 }, +] + +[[package]] +name = "click" +version = "8.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bb/63/f9e1ea081ce35720d8b92acde70daaedace594dc93b693c869e0d5910718/click-8.3.3.tar.gz", hash = "sha256:398329ad4837b2ff7cbe1dd166a4c0f8900c3ca3a218de04466f38f6497f18a2", size = 328061 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/44/c1221527f6a71a01ec6fbad7fa78f1d50dfa02217385cf0fa3eec7087d59/click-8.3.3-py3-none-any.whl", hash = "sha256:a2bf429bb3033c89fa4936ffb35d5cb471e3719e1f3c8a7c3fff0b8314305613", size = 110502 }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "cryptography" +version = "48.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/a9/db8f313fdcd85d767d4973515e1db101f9c71f95fced83233de224673757/cryptography-48.0.0.tar.gz", hash = "sha256:5c3932f4436d1cccb036cb0eaef46e6e2db91035166f1ad6505c3c9d5a635920", size = 832984 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/3d/01f6dd9190170a5a241e0e98c2d04be3664a9e6f5b9b872cde63aff1c3dd/cryptography-48.0.0-cp311-abi3-macosx_10_9_universal2.whl", hash = "sha256:0c558d2cdffd8f4bbb30fc7134c74d2ca9a476f830bb053074498fbc86f41ed6", size = 8001587 }, + { url = "https://files.pythonhosted.org/packages/b2/6e/e90527eef33f309beb811cf7c982c3aeffcce8e3edb178baa4ca3ae4a6fa/cryptography-48.0.0-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f5333311663ea94f75dd408665686aaf426563556bb5283554a3539177e03b8c", size = 4690433 }, + { url = "https://files.pythonhosted.org/packages/90/04/673510ed51ddff56575f306cf1617d80411ee76831ccd3097599140efdfe/cryptography-48.0.0-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7995ef305d7165c3f11ae07f2517e5a4f1d5c18da1376a0a9ed496336b69e5f3", size = 4710620 }, + { url = "https://files.pythonhosted.org/packages/14/d5/e9c4ef932c8d800490c34d8bd589d64a31d5890e27ec9e9ad532be893294/cryptography-48.0.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:40ba1f85eaa6959837b1d51c9767e230e14612eea4ef110ee8854ada22da1bf5", size = 4696283 }, + { url = "https://files.pythonhosted.org/packages/0c/29/174b9dfb60b12d59ecfc6cfa04bc88c21b42a54f01b8aae09bb6e51e4c7f/cryptography-48.0.0-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:369a6348999f94bbd53435c894377b20ab95f25a9065c283570e70150d8abc3c", size = 5296573 }, + { url = "https://files.pythonhosted.org/packages/95/38/0d29a6fd7d0d1373f0c0c88a04ba20e359b257753ac497564cd660fc1d55/cryptography-48.0.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:a0e692c683f4df67815a2d258b324e66f4738bd7a96a218c826dce4f4bd05d8f", size = 4743677 }, + { url = "https://files.pythonhosted.org/packages/30/be/eef653013d5c63b6a490529e0316f9ac14a37602965d4903efed1399f32b/cryptography-48.0.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:18349bbc56f4743c8b12dc32e2bccb2cf83ee8b69a3bba74ef8ae857e26b3d25", size = 4330808 }, + { url = "https://files.pythonhosted.org/packages/84/9e/500463e87abb7a0a0f9f256ec21123ecde0a7b5541a15e840ea54551fd81/cryptography-48.0.0-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:7e8eac43dfca5c4cccc6dad9a80504436fca53bb9bc3100a2386d730fbe6b602", size = 4695941 }, + { url = "https://files.pythonhosted.org/packages/e3/dc/7303087450c2ec9e7fbb750e17c2abfbc658f23cbd0e54009509b7cc4091/cryptography-48.0.0-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:9ccdac7d40688ecb5a3b4a604b8a88c8002e3442d6c60aead1db2a89a041560c", size = 5252579 }, + { url = "https://files.pythonhosted.org/packages/d0/c0/7101d3b7215edcdc90c45da544961fd8ed2d6448f77577460fa75a8443f7/cryptography-48.0.0-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:bd72e68b06bb1e96913f97dd4901119bc17f39d4586a5adf2d3e47bc2b9d58b5", size = 4743326 }, + { url = "https://files.pythonhosted.org/packages/ac/d8/5b833bad13016f562ab9d063d68199a4bd121d18458e439515601d3357ec/cryptography-48.0.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59baa2cb386c4f0b9905bd6eb4c2a79a69a128408fd31d32ca4d7102d4156321", size = 4826672 }, + { url = "https://files.pythonhosted.org/packages/98/e1/7074eb8bf3c135558c73fc2bcf0f5633f912e6fb87e868a55c454080ef09/cryptography-48.0.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9249e3cd978541d665967ac2cb2787fd6a62bddf1e75b3e347a594d7dacf4f74", size = 4972574 }, + { url = "https://files.pythonhosted.org/packages/04/70/e5a1b41d325f797f39427aa44ef8baf0be500065ab6d8e10369d850d4a4f/cryptography-48.0.0-cp311-abi3-win32.whl", hash = "sha256:9c459db21422be75e2809370b829a87eb37f74cd785fc4aa9ea1e5f43b47cda4", size = 3294868 }, + { url = "https://files.pythonhosted.org/packages/f4/ac/8ac51b4a5fc5932eb7ee5c517ba7dc8cd834f0048962b6b352f00f41ebf9/cryptography-48.0.0-cp311-abi3-win_amd64.whl", hash = "sha256:5b012212e08b8dd5edc78ef54da83dd9892fd9105323b3993eff6bea65dc21d7", size = 3817107 }, + { url = "https://files.pythonhosted.org/packages/6b/84/70e3feea9feea87fd7cbe77efb2712ae1e3e6edf10749dc6e95f4e60e455/cryptography-48.0.0-cp314-cp314t-macosx_10_9_universal2.whl", hash = "sha256:3cb07a3ed6431663cd321ea8a000a1314c74211f823e4177fefa2255e057d1ec", size = 7986556 }, + { url = "https://files.pythonhosted.org/packages/89/6e/18e07a618bb5442ba10cf4df16e99c071365528aa570dfcb8c02e25a303b/cryptography-48.0.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c7378637d7d88016fa6791c159f698b3d3eed28ebf844ac36b9dc04a14dae18", size = 4684776 }, + { url = "https://files.pythonhosted.org/packages/be/6a/4ea3b4c6c6759794d5ee2103c304a5076dc4b19ae1f9fe47dba439e159e9/cryptography-48.0.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc90c0b39b2e3c65ef52c804b72e3c58f8a04ab2a1871272798e5f9572c17d20", size = 4698121 }, + { url = "https://files.pythonhosted.org/packages/2f/59/6ff6ad6cae03bb887da2a5860b2c9805f8dac969ef01ce563336c49bd1d1/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:76341972e1eff8b4bea859f09c0d3e64b96ce931b084f9b9b7db8ef364c30eff", size = 4690042 }, + { url = "https://files.pythonhosted.org/packages/ca/b4/fc334ed8cfd705aca282fe4d8f5ae64a8e0f74932e9feecb344610cf6e4d/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:55b7718303bf06a5753dcdccf2f3945cf18ad7bffde41b61226e4db31ab89a9c", size = 5282526 }, + { url = "https://files.pythonhosted.org/packages/11/08/9f8c5386cc4cd90d8255c7cdd0f5baf459a08502a09de30dc51f553d38dc/cryptography-48.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:a64697c641c7b1b2178e573cbc31c7c6684cd56883a478d75143dbb7118036db", size = 4733116 }, + { url = "https://files.pythonhosted.org/packages/b8/77/99307d7574045699f8805aa500fa0fb83422d115b5400a064ddd306d7750/cryptography-48.0.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:561215ea3879cb1cbbf272867e2efda62476f240fb58c64de6b393ae19246741", size = 4316030 }, + { url = "https://files.pythonhosted.org/packages/fd/36/a608b98337af3cb2aff4818e406649d30572b7031918b04c87d979495348/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:ad64688338ed4bc1a6618076ba75fd7194a5f1797ac60b47afe926285adb3166", size = 4689640 }, + { url = "https://files.pythonhosted.org/packages/dd/a6/825010a291b4438aecc1f568bc428189fc1175515223632477c07dc0a6df/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:906cbf0670286c6e0044156bc7d4af9cbb0ef6db9f73e52c3ec56ba6bdde5336", size = 5237657 }, + { url = "https://files.pythonhosted.org/packages/b9/09/4e76a09b4caa29aad535ddc806f5d4c5d01885bd978bd984fbc6ca032cae/cryptography-48.0.0-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:ea8990436d914540a40ab24b6a77c0969695ed52f4a4874c5137ccf7045a7057", size = 4732362 }, + { url = "https://files.pythonhosted.org/packages/18/78/444fa04a77d0cb95f417dda20d450e13c56ba8e5220fc892a1658f44f882/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c18684a7f0cc9a3cb60328f496b8e3372def7c5d2df39ac267878b05565aaaae", size = 4819580 }, + { url = "https://files.pythonhosted.org/packages/38/85/ea67067c70a1fd4be2c63d35eeed82658023021affccc7b17705f8527dd2/cryptography-48.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:9be5aafa5736574f8f15f262adc81b2a9869e2cfe9014d52a44633905b40d52c", size = 4963283 }, + { url = "https://files.pythonhosted.org/packages/75/54/cc6d0f3deac3e81c7f847e8a189a12b6cdd65059b43dad25d4316abd849a/cryptography-48.0.0-cp314-cp314t-win32.whl", hash = "sha256:c17dfe85494deaeddc5ce251aebd1d60bbe6afc8b62071bb0b469431a000124f", size = 3270954 }, + { url = "https://files.pythonhosted.org/packages/49/67/cc947e288c0758a4e5473d1dcb743037ab7785541265a969240b8885441a/cryptography-48.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27241b1dc9962e056062a8eef1991d02c3a24569c95975bd2322a8a52c6e5e12", size = 3797313 }, + { url = "https://files.pythonhosted.org/packages/f2/63/61d4a4e1c6b6bab6ce1e213cd36a24c415d90e76d78c5eb8577c5541d2e8/cryptography-48.0.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:58d00498e8933e4a194f3076aee1b4a97dfec1a6da444535755822fe5d8b0b86", size = 7983482 }, + { url = "https://files.pythonhosted.org/packages/d5/ac/f5b5995b87770c693e2596559ffafe195b4033a57f14a82268a2842953f3/cryptography-48.0.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:614d0949f4790582d2cc25553abd09dd723025f0c0e7c67376a1d77196743d6e", size = 4683266 }, + { url = "https://files.pythonhosted.org/packages/ec/c6/8b14f67e18338fbc4adb76f66c001f5c3610b3e2d1837f268f47a347dbbb/cryptography-48.0.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ce4bfae76319a532a2dc68f82cc32f5676ee792a983187dac07183690e5c66f", size = 4696228 }, + { url = "https://files.pythonhosted.org/packages/ea/73/f808fbae9514bd91b47875b003f13e284c8c6bdfd904b7944e803937eec1/cryptography-48.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:2eb992bbd4661238c5a397594c83f5b4dc2bc5b848c365c8f991b6780efcc5c7", size = 4689097 }, + { url = "https://files.pythonhosted.org/packages/93/01/d86632d7d28db8ae83221995752eeb6639ffb374c2d22955648cf8d52797/cryptography-48.0.0-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:22a5cb272895dce158b2cacdfdc3debd299019659f42947dbdac6f32d68fe832", size = 5283582 }, + { url = "https://files.pythonhosted.org/packages/02/e1/50edc7a50334807cc4791fc4a0ce7468b4a1416d9138eab358bfc9a3d70b/cryptography-48.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:2b4d59804e8408e2fea7d1fbaf218e5ec984325221db76e6a241a9abd6cdd95c", size = 4730479 }, + { url = "https://files.pythonhosted.org/packages/6f/af/99a582b1b1641ff5911ac559beb45097cf79efd4ead4657f578ef1af2d47/cryptography-48.0.0-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:984a20b0f62a26f48a3396c72e4bc34c66e356d356bf370053066b3b6d54634a", size = 4326481 }, + { url = "https://files.pythonhosted.org/packages/90/ee/89aa26a06ef0a7d7611788ffd571a7c50e368cc6a4d5eef8b4884e866edb/cryptography-48.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:5a5ed8fde7a1d09376ca0b40e68cd59c69fe23b1f9768bd5824f54681626032a", size = 4688713 }, + { url = "https://files.pythonhosted.org/packages/70/ba/bcb1b0bb7a33d4c7c0c4d4c7874b4a62ae4f56113a5f4baefa362dfb1f0f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:8cd666227ef7af430aa5914a9910e0ddd703e75f039cef0825cd0da71b6b711a", size = 5238165 }, + { url = "https://files.pythonhosted.org/packages/c9/70/ca4003b1ce5ca3dc3186ada51908c8a9b9ff7d5cab83cc0d43ee14ec144f/cryptography-48.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9071196d81abc88b3516ac8cdfad32e2b66dd4a5393a8e68a961e9161ddc6239", size = 4729947 }, + { url = "https://files.pythonhosted.org/packages/44/a0/4ec7cf774207905aef1a8d11c3750d5a1db805eb380ee4e16df317870128/cryptography-48.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1e2d54c8be6152856a36f0882ab231e70f8ec7f14e93cf87db8a2ed056bf160c", size = 4822059 }, + { url = "https://files.pythonhosted.org/packages/1e/75/a2e55f99c16fcac7b5d6c1eb19ad8e00799854d6be5ca845f9259eae1681/cryptography-48.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a5da777e32ffed6f85a7b2b3f7c5cbc88c146bfcd0a1d7baf5fcc6c52ee35dd4", size = 4960575 }, + { url = "https://files.pythonhosted.org/packages/b8/23/6e6f32143ab5d8b36ca848a502c4bcd477ae75b9e1677e3530d669062578/cryptography-48.0.0-cp39-abi3-win32.whl", hash = "sha256:77a2ccbbe917f6710e05ba9adaa25fb5075620bf3ea6fb751997875aff4ae4bd", size = 3279117 }, + { url = "https://files.pythonhosted.org/packages/9d/9a/0fea98a70cf1749d41d738836f6349d97945f7c89433a259a6c2642eefeb/cryptography-48.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:16cd65b9330583e4619939b3a3843eec1e6e789744bb01e7c7e2e62e33c239c8", size = 3792100 }, + { url = "https://files.pythonhosted.org/packages/be/d2/024b5e06be9d44cb021fb0e1a03d34d63989cf56a0fe62f3dfbab695b9b4/cryptography-48.0.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:84cf79f0dc8b36ac5da873481716e87aef31fcfa0444f9e1d8b4b2cece142855", size = 3950391 }, + { url = "https://files.pythonhosted.org/packages/bc/17/3861e17c56fa0fd37491a14a8673fdb77c57fc5693cafe745ea8b06dba75/cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:fdfef35d751d510fcef5252703621574364fec16418c4a1e5e1055248401054b", size = 4637126 }, + { url = "https://files.pythonhosted.org/packages/f0/0a/7e226dbff530f21480727eb764973a7bff2b912f8e15cd4f129e71b56d1d/cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:0890f502ddf7d9c6426129c3f49f5c0a39278ed7cd6322c8755ffca6ee675a13", size = 4667270 }, + { url = "https://files.pythonhosted.org/packages/3b/f2/5a72274ca9f1b2a8b44a662ee0bf1b435909deb473d6f97bcd035bcdbc71/cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:ecde28a596bead48b0cfd2a1b4416c3d43074c2d785e3a398d7ec1fc4d0f7fbb", size = 4636797 }, + { url = "https://files.pythonhosted.org/packages/b4/e1/48cedb2fe63626e91ded1edad159e2a4fb8b6906c4425eb7749673077ce7/cryptography-48.0.0-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:4defde8685ae324a9eb9d818717e93b4638ef67070ac9bc15b8ca85f63048355", size = 4666800 }, + { url = "https://files.pythonhosted.org/packages/a2/ca/7e8365deec19afb2b2c7be7c1c0aa8f99633b54e90c570999acda93260fc/cryptography-48.0.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:db63bf618e5dea46c07de12e900fe1cdd2541e6dc9dbae772a70b7d4d4765f6a", size = 3739536 }, +] + +[[package]] +name = "deepeval" +version = "4.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aiohttp" }, + { name = "click" }, + { name = "grpcio" }, + { name = "jinja2" }, + { name = "nest-asyncio" }, + { name = "openai" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-sdk" }, + { name = "portalocker" }, + { name = "posthog" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyfiglet" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "pytest-repeat" }, + { name = "pytest-rerunfailures" }, + { name = "pytest-xdist" }, + { name = "python-dotenv" }, + { name = "requests" }, + { name = "rich" }, + { name = "sentry-sdk" }, + { name = "setuptools" }, + { name = "tabulate" }, + { name = "tenacity" }, + { name = "tqdm" }, + { name = "typer" }, + { name = "wheel" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/76/6713d8a7e311fd786d0c5c8e64dabad0946965b8f540ad2c481fbbd1f69d/deepeval-4.0.3.tar.gz", hash = "sha256:fbeec2272e69dd6a98638bba9db2b9653674fe2d94389b7b80b758563dc9df60", size = 680046 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/d3/3aeb104acfcf66be35be80ad6a5fd6511fc152fa2df6ac2702b43372b1df/deepeval-4.0.3-py3-none-any.whl", hash = "sha256:f3676b1228f4399348c9ba4b9653fef5402f74cd876f878437e16b10c511ac1b", size = 964114 }, +] + +[[package]] +name = "distro" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277 }, +] + +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708 }, +] + +[[package]] +name = "frozenlist" +version = "1.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/f5/c831fac6cc817d26fd54c7eaccd04ef7e0288806943f7cc5bbf69f3ac1f0/frozenlist-1.8.0.tar.gz", hash = "sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad", size = 45875 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bc/03/077f869d540370db12165c0aa51640a873fb661d8b315d1d4d67b284d7ac/frozenlist-1.8.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84", size = 86912 }, + { url = "https://files.pythonhosted.org/packages/df/b5/7610b6bd13e4ae77b96ba85abea1c8cb249683217ef09ac9e0ae93f25a91/frozenlist-1.8.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9", size = 50046 }, + { url = "https://files.pythonhosted.org/packages/6e/ef/0e8f1fe32f8a53dd26bdd1f9347efe0778b0fddf62789ea683f4cc7d787d/frozenlist-1.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93", size = 50119 }, + { url = "https://files.pythonhosted.org/packages/11/b1/71a477adc7c36e5fb628245dfbdea2166feae310757dea848d02bd0689fd/frozenlist-1.8.0-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f", size = 231067 }, + { url = "https://files.pythonhosted.org/packages/45/7e/afe40eca3a2dc19b9904c0f5d7edfe82b5304cb831391edec0ac04af94c2/frozenlist-1.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695", size = 233160 }, + { url = "https://files.pythonhosted.org/packages/a6/aa/7416eac95603ce428679d273255ffc7c998d4132cfae200103f164b108aa/frozenlist-1.8.0-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52", size = 228544 }, + { url = "https://files.pythonhosted.org/packages/8b/3d/2a2d1f683d55ac7e3875e4263d28410063e738384d3adc294f5ff3d7105e/frozenlist-1.8.0-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581", size = 243797 }, + { url = "https://files.pythonhosted.org/packages/78/1e/2d5565b589e580c296d3bb54da08d206e797d941a83a6fdea42af23be79c/frozenlist-1.8.0-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567", size = 247923 }, + { url = "https://files.pythonhosted.org/packages/aa/c3/65872fcf1d326a7f101ad4d86285c403c87be7d832b7470b77f6d2ed5ddc/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b", size = 230886 }, + { url = "https://files.pythonhosted.org/packages/a0/76/ac9ced601d62f6956f03cc794f9e04c81719509f85255abf96e2510f4265/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92", size = 245731 }, + { url = "https://files.pythonhosted.org/packages/b9/49/ecccb5f2598daf0b4a1415497eba4c33c1e8ce07495eb07d2860c731b8d5/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d", size = 241544 }, + { url = "https://files.pythonhosted.org/packages/53/4b/ddf24113323c0bbcc54cb38c8b8916f1da7165e07b8e24a717b4a12cbf10/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd", size = 241806 }, + { url = "https://files.pythonhosted.org/packages/a7/fb/9b9a084d73c67175484ba2789a59f8eebebd0827d186a8102005ce41e1ba/frozenlist-1.8.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967", size = 229382 }, + { url = "https://files.pythonhosted.org/packages/95/a3/c8fb25aac55bf5e12dae5c5aa6a98f85d436c1dc658f21c3ac73f9fa95e5/frozenlist-1.8.0-cp311-cp311-win32.whl", hash = "sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25", size = 39647 }, + { url = "https://files.pythonhosted.org/packages/0a/f5/603d0d6a02cfd4c8f2a095a54672b3cf967ad688a60fb9faf04fc4887f65/frozenlist-1.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b", size = 44064 }, + { url = "https://files.pythonhosted.org/packages/5d/16/c2c9ab44e181f043a86f9a8f84d5124b62dbcb3a02c0977ec72b9ac1d3e0/frozenlist-1.8.0-cp311-cp311-win_arm64.whl", hash = "sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a", size = 39937 }, + { url = "https://files.pythonhosted.org/packages/69/29/948b9aa87e75820a38650af445d2ef2b6b8a6fab1a23b6bb9e4ef0be2d59/frozenlist-1.8.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1", size = 87782 }, + { url = "https://files.pythonhosted.org/packages/64/80/4f6e318ee2a7c0750ed724fa33a4bdf1eacdc5a39a7a24e818a773cd91af/frozenlist-1.8.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b", size = 50594 }, + { url = "https://files.pythonhosted.org/packages/2b/94/5c8a2b50a496b11dd519f4a24cb5496cf125681dd99e94c604ccdea9419a/frozenlist-1.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4", size = 50448 }, + { url = "https://files.pythonhosted.org/packages/6a/bd/d91c5e39f490a49df14320f4e8c80161cfcce09f1e2cde1edd16a551abb3/frozenlist-1.8.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383", size = 242411 }, + { url = "https://files.pythonhosted.org/packages/8f/83/f61505a05109ef3293dfb1ff594d13d64a2324ac3482be2cedc2be818256/frozenlist-1.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4", size = 243014 }, + { url = "https://files.pythonhosted.org/packages/d8/cb/cb6c7b0f7d4023ddda30cf56b8b17494eb3a79e3fda666bf735f63118b35/frozenlist-1.8.0-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8", size = 234909 }, + { url = "https://files.pythonhosted.org/packages/31/c5/cd7a1f3b8b34af009fb17d4123c5a778b44ae2804e3ad6b86204255f9ec5/frozenlist-1.8.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b", size = 250049 }, + { url = "https://files.pythonhosted.org/packages/c0/01/2f95d3b416c584a1e7f0e1d6d31998c4a795f7544069ee2e0962a4b60740/frozenlist-1.8.0-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52", size = 256485 }, + { url = "https://files.pythonhosted.org/packages/ce/03/024bf7720b3abaebcff6d0793d73c154237b85bdf67b7ed55e5e9596dc9a/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29", size = 237619 }, + { url = "https://files.pythonhosted.org/packages/69/fa/f8abdfe7d76b731f5d8bd217827cf6764d4f1d9763407e42717b4bed50a0/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3", size = 250320 }, + { url = "https://files.pythonhosted.org/packages/f5/3c/b051329f718b463b22613e269ad72138cc256c540f78a6de89452803a47d/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143", size = 246820 }, + { url = "https://files.pythonhosted.org/packages/0f/ae/58282e8f98e444b3f4dd42448ff36fa38bef29e40d40f330b22e7108f565/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608", size = 250518 }, + { url = "https://files.pythonhosted.org/packages/8f/96/007e5944694d66123183845a106547a15944fbbb7154788cbf7272789536/frozenlist-1.8.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa", size = 239096 }, + { url = "https://files.pythonhosted.org/packages/66/bb/852b9d6db2fa40be96f29c0d1205c306288f0684df8fd26ca1951d461a56/frozenlist-1.8.0-cp312-cp312-win32.whl", hash = "sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf", size = 39985 }, + { url = "https://files.pythonhosted.org/packages/b8/af/38e51a553dd66eb064cdf193841f16f077585d4d28394c2fa6235cb41765/frozenlist-1.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746", size = 44591 }, + { url = "https://files.pythonhosted.org/packages/a7/06/1dc65480ab147339fecc70797e9c2f69d9cea9cf38934ce08df070fdb9cb/frozenlist-1.8.0-cp312-cp312-win_arm64.whl", hash = "sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd", size = 40102 }, + { url = "https://files.pythonhosted.org/packages/2d/40/0832c31a37d60f60ed79e9dfb5a92e1e2af4f40a16a29abcc7992af9edff/frozenlist-1.8.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a", size = 85717 }, + { url = "https://files.pythonhosted.org/packages/30/ba/b0b3de23f40bc55a7057bd38434e25c34fa48e17f20ee273bbde5e0650f3/frozenlist-1.8.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7", size = 49651 }, + { url = "https://files.pythonhosted.org/packages/0c/ab/6e5080ee374f875296c4243c381bbdef97a9ac39c6e3ce1d5f7d42cb78d6/frozenlist-1.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40", size = 49417 }, + { url = "https://files.pythonhosted.org/packages/d5/4e/e4691508f9477ce67da2015d8c00acd751e6287739123113a9fca6f1604e/frozenlist-1.8.0-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027", size = 234391 }, + { url = "https://files.pythonhosted.org/packages/40/76/c202df58e3acdf12969a7895fd6f3bc016c642e6726aa63bd3025e0fc71c/frozenlist-1.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822", size = 233048 }, + { url = "https://files.pythonhosted.org/packages/f9/c0/8746afb90f17b73ca5979c7a3958116e105ff796e718575175319b5bb4ce/frozenlist-1.8.0-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121", size = 226549 }, + { url = "https://files.pythonhosted.org/packages/7e/eb/4c7eefc718ff72f9b6c4893291abaae5fbc0c82226a32dcd8ef4f7a5dbef/frozenlist-1.8.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5", size = 239833 }, + { url = "https://files.pythonhosted.org/packages/c2/4e/e5c02187cf704224f8b21bee886f3d713ca379535f16893233b9d672ea71/frozenlist-1.8.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e", size = 245363 }, + { url = "https://files.pythonhosted.org/packages/1f/96/cb85ec608464472e82ad37a17f844889c36100eed57bea094518bf270692/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11", size = 229314 }, + { url = "https://files.pythonhosted.org/packages/5d/6f/4ae69c550e4cee66b57887daeebe006fe985917c01d0fff9caab9883f6d0/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1", size = 243365 }, + { url = "https://files.pythonhosted.org/packages/7a/58/afd56de246cf11780a40a2c28dc7cbabbf06337cc8ddb1c780a2d97e88d8/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1", size = 237763 }, + { url = "https://files.pythonhosted.org/packages/cb/36/cdfaf6ed42e2644740d4a10452d8e97fa1c062e2a8006e4b09f1b5fd7d63/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8", size = 240110 }, + { url = "https://files.pythonhosted.org/packages/03/a8/9ea226fbefad669f11b52e864c55f0bd57d3c8d7eb07e9f2e9a0b39502e1/frozenlist-1.8.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed", size = 233717 }, + { url = "https://files.pythonhosted.org/packages/1e/0b/1b5531611e83ba7d13ccc9988967ea1b51186af64c42b7a7af465dcc9568/frozenlist-1.8.0-cp313-cp313-win32.whl", hash = "sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496", size = 39628 }, + { url = "https://files.pythonhosted.org/packages/d8/cf/174c91dbc9cc49bc7b7aab74d8b734e974d1faa8f191c74af9b7e80848e6/frozenlist-1.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231", size = 43882 }, + { url = "https://files.pythonhosted.org/packages/c1/17/502cd212cbfa96eb1388614fe39a3fc9ab87dbbe042b66f97acb57474834/frozenlist-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62", size = 39676 }, + { url = "https://files.pythonhosted.org/packages/d2/5c/3bbfaa920dfab09e76946a5d2833a7cbdf7b9b4a91c714666ac4855b88b4/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94", size = 89235 }, + { url = "https://files.pythonhosted.org/packages/d2/d6/f03961ef72166cec1687e84e8925838442b615bd0b8854b54923ce5b7b8a/frozenlist-1.8.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c", size = 50742 }, + { url = "https://files.pythonhosted.org/packages/1e/bb/a6d12b7ba4c3337667d0e421f7181c82dda448ce4e7ad7ecd249a16fa806/frozenlist-1.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52", size = 51725 }, + { url = "https://files.pythonhosted.org/packages/bc/71/d1fed0ffe2c2ccd70b43714c6cab0f4188f09f8a67a7914a6b46ee30f274/frozenlist-1.8.0-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51", size = 284533 }, + { url = "https://files.pythonhosted.org/packages/c9/1f/fb1685a7b009d89f9bf78a42d94461bc06581f6e718c39344754a5d9bada/frozenlist-1.8.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65", size = 292506 }, + { url = "https://files.pythonhosted.org/packages/e6/3b/b991fe1612703f7e0d05c0cf734c1b77aaf7c7d321df4572e8d36e7048c8/frozenlist-1.8.0-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82", size = 274161 }, + { url = "https://files.pythonhosted.org/packages/ca/ec/c5c618767bcdf66e88945ec0157d7f6c4a1322f1473392319b7a2501ded7/frozenlist-1.8.0-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714", size = 294676 }, + { url = "https://files.pythonhosted.org/packages/7c/ce/3934758637d8f8a88d11f0585d6495ef54b2044ed6ec84492a91fa3b27aa/frozenlist-1.8.0-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d", size = 300638 }, + { url = "https://files.pythonhosted.org/packages/fc/4f/a7e4d0d467298f42de4b41cbc7ddaf19d3cfeabaf9ff97c20c6c7ee409f9/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506", size = 283067 }, + { url = "https://files.pythonhosted.org/packages/dc/48/c7b163063d55a83772b268e6d1affb960771b0e203b632cfe09522d67ea5/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51", size = 292101 }, + { url = "https://files.pythonhosted.org/packages/9f/d0/2366d3c4ecdc2fd391e0afa6e11500bfba0ea772764d631bbf82f0136c9d/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e", size = 289901 }, + { url = "https://files.pythonhosted.org/packages/b8/94/daff920e82c1b70e3618a2ac39fbc01ae3e2ff6124e80739ce5d71c9b920/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0", size = 289395 }, + { url = "https://files.pythonhosted.org/packages/e3/20/bba307ab4235a09fdcd3cc5508dbabd17c4634a1af4b96e0f69bfe551ebd/frozenlist-1.8.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41", size = 283659 }, + { url = "https://files.pythonhosted.org/packages/fd/00/04ca1c3a7a124b6de4f8a9a17cc2fcad138b4608e7a3fc5877804b8715d7/frozenlist-1.8.0-cp313-cp313t-win32.whl", hash = "sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b", size = 43492 }, + { url = "https://files.pythonhosted.org/packages/59/5e/c69f733a86a94ab10f68e496dc6b7e8bc078ebb415281d5698313e3af3a1/frozenlist-1.8.0-cp313-cp313t-win_amd64.whl", hash = "sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888", size = 48034 }, + { url = "https://files.pythonhosted.org/packages/16/6c/be9d79775d8abe79b05fa6d23da99ad6e7763a1d080fbae7290b286093fd/frozenlist-1.8.0-cp313-cp313t-win_arm64.whl", hash = "sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042", size = 41749 }, + { url = "https://files.pythonhosted.org/packages/f1/c8/85da824b7e7b9b6e7f7705b2ecaf9591ba6f79c1177f324c2735e41d36a2/frozenlist-1.8.0-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0", size = 86127 }, + { url = "https://files.pythonhosted.org/packages/8e/e8/a1185e236ec66c20afd72399522f142c3724c785789255202d27ae992818/frozenlist-1.8.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f", size = 49698 }, + { url = "https://files.pythonhosted.org/packages/a1/93/72b1736d68f03fda5fdf0f2180fb6caaae3894f1b854d006ac61ecc727ee/frozenlist-1.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c", size = 49749 }, + { url = "https://files.pythonhosted.org/packages/a7/b2/fabede9fafd976b991e9f1b9c8c873ed86f202889b864756f240ce6dd855/frozenlist-1.8.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2", size = 231298 }, + { url = "https://files.pythonhosted.org/packages/3a/3b/d9b1e0b0eed36e70477ffb8360c49c85c8ca8ef9700a4e6711f39a6e8b45/frozenlist-1.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8", size = 232015 }, + { url = "https://files.pythonhosted.org/packages/dc/94/be719d2766c1138148564a3960fc2c06eb688da592bdc25adcf856101be7/frozenlist-1.8.0-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686", size = 225038 }, + { url = "https://files.pythonhosted.org/packages/e4/09/6712b6c5465f083f52f50cf74167b92d4ea2f50e46a9eea0523d658454ae/frozenlist-1.8.0-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e", size = 240130 }, + { url = "https://files.pythonhosted.org/packages/f8/d4/cd065cdcf21550b54f3ce6a22e143ac9e4836ca42a0de1022da8498eac89/frozenlist-1.8.0-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a", size = 242845 }, + { url = "https://files.pythonhosted.org/packages/62/c3/f57a5c8c70cd1ead3d5d5f776f89d33110b1addae0ab010ad774d9a44fb9/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128", size = 229131 }, + { url = "https://files.pythonhosted.org/packages/6c/52/232476fe9cb64f0742f3fde2b7d26c1dac18b6d62071c74d4ded55e0ef94/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f", size = 240542 }, + { url = "https://files.pythonhosted.org/packages/5f/85/07bf3f5d0fb5414aee5f47d33c6f5c77bfe49aac680bfece33d4fdf6a246/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7", size = 237308 }, + { url = "https://files.pythonhosted.org/packages/11/99/ae3a33d5befd41ac0ca2cc7fd3aa707c9c324de2e89db0e0f45db9a64c26/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30", size = 238210 }, + { url = "https://files.pythonhosted.org/packages/b2/60/b1d2da22f4970e7a155f0adde9b1435712ece01b3cd45ba63702aea33938/frozenlist-1.8.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7", size = 231972 }, + { url = "https://files.pythonhosted.org/packages/3f/ab/945b2f32de889993b9c9133216c068b7fcf257d8595a0ac420ac8677cab0/frozenlist-1.8.0-cp314-cp314-win32.whl", hash = "sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806", size = 40536 }, + { url = "https://files.pythonhosted.org/packages/59/ad/9caa9b9c836d9ad6f067157a531ac48b7d36499f5036d4141ce78c230b1b/frozenlist-1.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0", size = 44330 }, + { url = "https://files.pythonhosted.org/packages/82/13/e6950121764f2676f43534c555249f57030150260aee9dcf7d64efda11dd/frozenlist-1.8.0-cp314-cp314-win_arm64.whl", hash = "sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b", size = 40627 }, + { url = "https://files.pythonhosted.org/packages/c0/c7/43200656ecc4e02d3f8bc248df68256cd9572b3f0017f0a0c4e93440ae23/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d", size = 89238 }, + { url = "https://files.pythonhosted.org/packages/d1/29/55c5f0689b9c0fb765055629f472c0de484dcaf0acee2f7707266ae3583c/frozenlist-1.8.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed", size = 50738 }, + { url = "https://files.pythonhosted.org/packages/ba/7d/b7282a445956506fa11da8c2db7d276adcbf2b17d8bb8407a47685263f90/frozenlist-1.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930", size = 51739 }, + { url = "https://files.pythonhosted.org/packages/62/1c/3d8622e60d0b767a5510d1d3cf21065b9db874696a51ea6d7a43180a259c/frozenlist-1.8.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c", size = 284186 }, + { url = "https://files.pythonhosted.org/packages/2d/14/aa36d5f85a89679a85a1d44cd7a6657e0b1c75f61e7cad987b203d2daca8/frozenlist-1.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24", size = 292196 }, + { url = "https://files.pythonhosted.org/packages/05/23/6bde59eb55abd407d34f77d39a5126fb7b4f109a3f611d3929f14b700c66/frozenlist-1.8.0-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37", size = 273830 }, + { url = "https://files.pythonhosted.org/packages/d2/3f/22cff331bfad7a8afa616289000ba793347fcd7bc275f3b28ecea2a27909/frozenlist-1.8.0-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a", size = 294289 }, + { url = "https://files.pythonhosted.org/packages/a4/89/5b057c799de4838b6c69aa82b79705f2027615e01be996d2486a69ca99c4/frozenlist-1.8.0-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2", size = 300318 }, + { url = "https://files.pythonhosted.org/packages/30/de/2c22ab3eb2a8af6d69dc799e48455813bab3690c760de58e1bf43b36da3e/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef", size = 282814 }, + { url = "https://files.pythonhosted.org/packages/59/f7/970141a6a8dbd7f556d94977858cfb36fa9b66e0892c6dd780d2219d8cd8/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe", size = 291762 }, + { url = "https://files.pythonhosted.org/packages/c1/15/ca1adae83a719f82df9116d66f5bb28bb95557b3951903d39135620ef157/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8", size = 289470 }, + { url = "https://files.pythonhosted.org/packages/ac/83/dca6dc53bf657d371fbc88ddeb21b79891e747189c5de990b9dfff2ccba1/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a", size = 289042 }, + { url = "https://files.pythonhosted.org/packages/96/52/abddd34ca99be142f354398700536c5bd315880ed0a213812bc491cff5e4/frozenlist-1.8.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e", size = 283148 }, + { url = "https://files.pythonhosted.org/packages/af/d3/76bd4ed4317e7119c2b7f57c3f6934aba26d277acc6309f873341640e21f/frozenlist-1.8.0-cp314-cp314t-win32.whl", hash = "sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df", size = 44676 }, + { url = "https://files.pythonhosted.org/packages/89/76/c615883b7b521ead2944bb3480398cbb07e12b7b4e4d073d3752eb721558/frozenlist-1.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd", size = 49451 }, + { url = "https://files.pythonhosted.org/packages/e0/a3/5982da14e113d07b325230f95060e2169f5311b1017ea8af2a29b374c289/frozenlist-1.8.0-cp314-cp314t-win_arm64.whl", hash = "sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79", size = 42507 }, + { url = "https://files.pythonhosted.org/packages/9a/9a/e35b4a917281c0b8419d4207f4334c8e8c5dbf4f3f5f9ada73958d937dcc/frozenlist-1.8.0-py3-none-any.whl", hash = "sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d", size = 13409 }, +] + +[[package]] +name = "grpcio" +version = "1.80.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/48/af6173dbca4454f4637a4678b67f52ca7e0c1ed7d5894d89d434fecede05/grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257", size = 12978905 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/db/1d56e5f5823257b291962d6c0ce106146c6447f405b60b234c4f222a7cde/grpcio-1.80.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:dfab85db094068ff42e2a3563f60ab3dddcc9d6488a35abf0132daec13209c8a", size = 6055009 }, + { url = "https://files.pythonhosted.org/packages/6e/18/c83f3cad64c5ca63bca7e91e5e46b0d026afc5af9d0a9972472ceba294b3/grpcio-1.80.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:5c07e82e822e1161354e32da2662f741a4944ea955f9f580ec8fb409dd6f6060", size = 12035295 }, + { url = "https://files.pythonhosted.org/packages/0f/8e/e14966b435be2dda99fbe89db9525ea436edc79780431a1c2875a3582644/grpcio-1.80.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ba0915d51fd4ced2db5ff719f84e270afe0e2d4c45a7bdb1e8d036e4502928c2", size = 6610297 }, + { url = "https://files.pythonhosted.org/packages/cc/26/d5eb38f42ce0e3fdc8174ea4d52036ef8d58cc4426cb800f2610f625dd75/grpcio-1.80.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:3cb8130ba457d2aa09fa6b7c3ed6b6e4e6a2685fce63cb803d479576c4d80e21", size = 7300208 }, + { url = "https://files.pythonhosted.org/packages/25/51/bd267c989f85a17a5b3eea65a6feb4ff672af41ca614e5a0279cc0ea381c/grpcio-1.80.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:09e5e478b3d14afd23f12e49e8b44c8684ac3c5f08561c43a5b9691c54d136ab", size = 6813442 }, + { url = "https://files.pythonhosted.org/packages/9e/d9/d80eef735b19e9169e30164bbf889b46f9df9127598a83d174eb13a48b26/grpcio-1.80.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:00168469238b022500e486c1c33916acf2f2a9b2c022202cf8a1885d2e3073c1", size = 7414743 }, + { url = "https://files.pythonhosted.org/packages/de/f2/567f5bd5054398ed6b0509b9a30900376dcf2786bd936812098808b49d8d/grpcio-1.80.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8502122a3cc1714038e39a0b071acb1207ca7844208d5ea0d091317555ee7106", size = 8426046 }, + { url = "https://files.pythonhosted.org/packages/62/29/73ef0141b4732ff5eacd68430ff2512a65c004696997f70476a83e548e7e/grpcio-1.80.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ce1794f4ea6cc3ca29463f42d665c32ba1b964b48958a66497917fe9069f26e6", size = 7851641 }, + { url = "https://files.pythonhosted.org/packages/46/69/abbfa360eb229a8623bab5f5a4f8105e445bd38ce81a89514ba55d281ad0/grpcio-1.80.0-cp311-cp311-win32.whl", hash = "sha256:51b4a7189b0bef2aa30adce3c78f09c83526cf3dddb24c6a96555e3b97340440", size = 4154368 }, + { url = "https://files.pythonhosted.org/packages/6f/d4/ae92206d01183b08613e846076115f5ac5991bae358d2a749fa864da5699/grpcio-1.80.0-cp311-cp311-win_amd64.whl", hash = "sha256:02e64bb0bb2da14d947a49e6f120a75e947250aebe65f9629b62bb1f5c14e6e9", size = 4894235 }, + { url = "https://files.pythonhosted.org/packages/5c/e8/a2b749265eb3415abc94f2e619bbd9e9707bebdda787e61c593004ec927a/grpcio-1.80.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:c624cc9f1008361014378c9d776de7182b11fe8b2e5a81bc69f23a295f2a1ad0", size = 6015616 }, + { url = "https://files.pythonhosted.org/packages/3e/97/b1282161a15d699d1e90c360df18d19165a045ce1c343c7f313f5e8a0b77/grpcio-1.80.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:f49eddcac43c3bf350c0385366a58f36bed8cc2c0ec35ef7b74b49e56552c0c2", size = 12014204 }, + { url = "https://files.pythonhosted.org/packages/6e/5e/d319c6e997b50c155ac5a8cb12f5173d5b42677510e886d250d50264949d/grpcio-1.80.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d334591df610ab94714048e0d5b4f3dd5ad1bee74dfec11eee344220077a79de", size = 6563866 }, + { url = "https://files.pythonhosted.org/packages/ae/f6/fdd975a2cb4d78eb67769a7b3b3830970bfa2e919f1decf724ae4445f42c/grpcio-1.80.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:0cb517eb1d0d0aaf1d87af7cc5b801d686557c1d88b2619f5e31fab3c2315921", size = 7273060 }, + { url = "https://files.pythonhosted.org/packages/db/f0/a3deb5feba60d9538a962913e37bd2e69a195f1c3376a3dd44fe0427e996/grpcio-1.80.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4e78c4ac0d97dc2e569b2f4bcbbb447491167cb358d1a389fc4af71ab6f70411", size = 6782121 }, + { url = "https://files.pythonhosted.org/packages/ca/84/36c6dcfddc093e108141f757c407902a05085e0c328007cb090d56646cdf/grpcio-1.80.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2ed770b4c06984f3b47eb0517b1c69ad0b84ef3f40128f51448433be904634cd", size = 7383811 }, + { url = "https://files.pythonhosted.org/packages/7c/ef/f3a77e3dc5b471a0ec86c564c98d6adfa3510d38f8ee99010410858d591e/grpcio-1.80.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:256507e2f524092f1473071a05e65a5b10d84b82e3ff24c5b571513cfaa61e2f", size = 8393860 }, + { url = "https://files.pythonhosted.org/packages/9b/8d/9d4d27ed7f33d109c50d6b5ce578a9914aa68edab75d65869a17e630a8d1/grpcio-1.80.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:9a6284a5d907c37db53350645567c522be314bac859a64a7a5ca63b77bb7958f", size = 7830132 }, + { url = "https://files.pythonhosted.org/packages/14/e4/9990b41c6d7a44e1e9dee8ac11d7a9802ba1378b40d77468a7761d1ad288/grpcio-1.80.0-cp312-cp312-win32.whl", hash = "sha256:c71309cfce2f22be26aa4a847357c502db6c621f1a49825ae98aa0907595b193", size = 4140904 }, + { url = "https://files.pythonhosted.org/packages/2f/2c/296f6138caca1f4b92a31ace4ae1b87dab692fc16a7a3417af3bb3c805bf/grpcio-1.80.0-cp312-cp312-win_amd64.whl", hash = "sha256:9fe648599c0e37594c4809d81a9e77bd138cc82eb8baa71b6a86af65426723ff", size = 4880944 }, + { url = "https://files.pythonhosted.org/packages/2f/3a/7c3c25789e3f069e581dc342e03613c5b1cb012c4e8c7d9d5cf960a75856/grpcio-1.80.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:e9e408fc016dffd20661f0126c53d8a31c2821b5c13c5d67a0f5ed5de93319ad", size = 6017243 }, + { url = "https://files.pythonhosted.org/packages/04/19/21a9806eb8240e174fd1ab0cd5b9aa948bb0e05c2f2f55f9d5d7405e6d08/grpcio-1.80.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:92d787312e613754d4d8b9ca6d3297e69994a7912a32fa38c4c4e01c272974b0", size = 12010840 }, + { url = "https://files.pythonhosted.org/packages/18/3a/23347d35f76f639e807fb7a36fad3068aed100996849a33809591f26eca6/grpcio-1.80.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac393b58aa16991a2f1144ec578084d544038c12242da3a215966b512904d0f", size = 6567644 }, + { url = "https://files.pythonhosted.org/packages/ff/40/96e07ecb604a6a67ae6ab151e3e35b132875d98bc68ec65f3e5ab3e781d7/grpcio-1.80.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:68e5851ac4b9afe07e7f84483803ad167852570d65326b34d54ca560bfa53fb6", size = 7277830 }, + { url = "https://files.pythonhosted.org/packages/9b/e2/da1506ecea1f34a5e365964644b35edef53803052b763ca214ba3870c856/grpcio-1.80.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:873ff5d17d68992ef6605330127425d2fc4e77e612fa3c3e0ed4e668685e3140", size = 6783216 }, + { url = "https://files.pythonhosted.org/packages/44/83/3b20ff58d0c3b7f6caaa3af9a4174d4023701df40a3f39f7f1c8e7c48f9d/grpcio-1.80.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:2bea16af2750fd0a899bf1abd9022244418b55d1f37da2202249ba4ba673838d", size = 7385866 }, + { url = "https://files.pythonhosted.org/packages/47/45/55c507599c5520416de5eefecc927d6a0d7af55e91cfffb2e410607e5744/grpcio-1.80.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba0db34f7e1d803a878284cd70e4c63cb6ae2510ba51937bf8f45ba997cefcf7", size = 8391602 }, + { url = "https://files.pythonhosted.org/packages/10/bb/dd06f4c24c01db9cf11341b547d0a016b2c90ed7dbbb086a5710df7dd1d7/grpcio-1.80.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8eb613f02d34721f1acf3626dfdb3545bd3c8505b0e52bf8b5710a28d02e8aa7", size = 7826752 }, + { url = "https://files.pythonhosted.org/packages/f9/1e/9d67992ba23371fd63d4527096eb8c6b76d74d52b500df992a3343fd7251/grpcio-1.80.0-cp313-cp313-win32.whl", hash = "sha256:93b6f823810720912fd131f561f91f5fed0fda372b6b7028a2681b8194d5d294", size = 4142310 }, + { url = "https://files.pythonhosted.org/packages/cf/e6/283326a27da9e2c3038bc93eeea36fb118ce0b2d03922a9cda6688f53c5b/grpcio-1.80.0-cp313-cp313-win_amd64.whl", hash = "sha256:e172cf795a3ba5246d3529e4d34c53db70e888fa582a8ffebd2e6e48bc0cba50", size = 4882833 }, + { url = "https://files.pythonhosted.org/packages/c5/6d/e65307ce20f5a09244ba9e9d8476e99fb039de7154f37fb85f26978b59c3/grpcio-1.80.0-cp314-cp314-linux_armv7l.whl", hash = "sha256:3d4147a97c8344d065d01bbf8b6acec2cf86fb0400d40696c8bdad34a64ffc0e", size = 6017376 }, + { url = "https://files.pythonhosted.org/packages/69/10/9cef5d9650c72625a699c549940f0abb3c4bfdb5ed45a5ce431f92f31806/grpcio-1.80.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d8e11f167935b3eb089ac9038e1a063e6d7dbe995c0bb4a661e614583352e76f", size = 12018133 }, + { url = "https://files.pythonhosted.org/packages/04/82/983aabaad82ba26113caceeb9091706a0696b25da004fe3defb5b346e15b/grpcio-1.80.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f14b618fc30de822681ee986cfdcc2d9327229dc4c98aed16896761cacd468b9", size = 6574748 }, + { url = "https://files.pythonhosted.org/packages/07/d7/031666ef155aa0bf399ed7e19439656c38bbd143779ae0861b038ce82abd/grpcio-1.80.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:4ed39fbdcf9b87370f6e8df4e39ca7b38b3e5e9d1b0013c7b6be9639d6578d14", size = 7277711 }, + { url = "https://files.pythonhosted.org/packages/e8/43/f437a78f7f4f1d311804189e8f11fb311a01049b2e08557c1068d470cb2e/grpcio-1.80.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2dcc70e9f0ba987526e8e8603a610fb4f460e42899e74e7a518bf3c68fe1bf05", size = 6785372 }, + { url = "https://files.pythonhosted.org/packages/93/3d/f6558e9c6296cb4227faa5c43c54a34c68d32654b829f53288313d16a86e/grpcio-1.80.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:448c884b668b868562b1bda833c5fce6272d26e1926ec46747cda05741d302c1", size = 7395268 }, + { url = "https://files.pythonhosted.org/packages/06/21/0fdd77e84720b08843c371a2efa6f2e19dbebf56adc72df73d891f5506f0/grpcio-1.80.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a1dc80fe55685b4a543555e6eef975303b36c8db1023b1599b094b92aa77965f", size = 8392000 }, + { url = "https://files.pythonhosted.org/packages/f5/68/67f4947ed55d2e69f2cc199ab9fd85e0a0034d813bbeef84df6d2ba4d4b7/grpcio-1.80.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:31b9ac4ad1aa28ffee5503821fafd09e4da0a261ce1c1281c6c8da0423c83b6e", size = 7828477 }, + { url = "https://files.pythonhosted.org/packages/44/b6/8d4096691b2e385e8271911a0de4f35f0a6c7d05aff7098e296c3de86939/grpcio-1.80.0-cp314-cp314-win32.whl", hash = "sha256:367ce30ba67d05e0592470428f0ec1c31714cab9ef19b8f2e37be1f4c7d32fae", size = 4218563 }, + { url = "https://files.pythonhosted.org/packages/e5/8c/bbe6baf2557262834f2070cf668515fa308b2d38a4bbf771f8f7872a7036/grpcio-1.80.0-cp314-cp314-win_amd64.whl", hash = "sha256:3b01e1f5464c583d2f567b2e46ff0d516ef979978f72091fd81f5ab7fa6e2e7f", size = 5019457 }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515 }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784 }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 }, +] + +[[package]] +name = "httpx-sse" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960 }, +] + +[[package]] +name = "idna" +version = "3.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/88/bcf9709822fe69d02c2a6a77956c98ce6ea8ca8767a9aadcedc7eb6a2390/idna-3.16.tar.gz", hash = "sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d", size = 203770 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/16/70255075a9859a0e3adb789b68ceb0e210dec03934245fd98d248226572f/idna-3.16-py3-none-any.whl", hash = "sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5", size = 74165 }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484 }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899 }, +] + +[[package]] +name = "jiter" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/b5/55f06bb281d92fb3cc86d14e1def2bd908bb77693183e7cb1f5a3c388b0c/jiter-0.15.0.tar.gz", hash = "sha256:4251acc80e2b7c9b7b8823456ea0fceeb0734dac2df7636d3c711b38476b5a76", size = 166640 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/13/daa722f5765c393576f466378f9dfd29d77c9bed939e0688f96afa3601ea/jiter-0.15.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0f862193b8696249d22ec433e85fd2ab0ad9596bc3e45e6c0bc55e8aeba97be2", size = 310899 }, + { url = "https://files.pythonhosted.org/packages/7f/82/2d2551829b082f4b6d82b9f939b031fb808a10aab1ec0664f82e150bb9a2/jiter-0.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1303d4d68a9b051ea90502402063ecf3807da00ad2affa19ca1ae3b90b3c5f67", size = 314963 }, + { url = "https://files.pythonhosted.org/packages/2a/0a/8b1a51466f7fe9f31dbe4bc7e0ca848674f9825e0f737b929b97e8c60aa7/jiter-0.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:392b8ab019e5502d08aff85c6272209c24bc2cbe706ea82a56368f524236614a", size = 341730 }, + { url = "https://files.pythonhosted.org/packages/f6/2a/e71dea19822e2e404e83992a08c1d6b9b617bb944f28c9c2fbd85d02c91e/jiter-0.15.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:773b6eb282ce11ee19f05f6b2d4404fa308e5bbd353b0b80a0262caad6db2cd7", size = 366214 }, + { url = "https://files.pythonhosted.org/packages/c4/59/97e1fa539d124a509a00ab7f669289d1c1d236ecabf12948a18f16c91082/jiter-0.15.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8d2c0c44d569ce0f2850f5c926f8caeb5f245fbc84475aeb36efccc2103e6dbd", size = 459527 }, + { url = "https://files.pythonhosted.org/packages/d1/7a/4a68d331aef8cf2e2393c14a3aacb635c62aa86071b0229899fb5baaa907/jiter-0.15.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:032396229564bca02440396bd327710719f724f5e7b7e9f7a8eb3faa4a2c2281", size = 375451 }, + { url = "https://files.pythonhosted.org/packages/7b/7e/1c445c2b6f0e30a274dc8082e0c3c7825411cce80d726bccd697c98cc8d3/jiter-0.15.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3d37768fce7f88dd2a8c6091f2325dea27d30d30d5c6e7a1c0f0af77723b708", size = 349428 }, + { url = "https://files.pythonhosted.org/packages/00/94/e20d38984fc17a636371bffd2ae0f698124fdc8e75ef969cd2da6ba7cea7/jiter-0.15.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:2c9cb907439d20bd0c7d7565ca01ee52234203208433749bae5b516907526928", size = 355405 }, + { url = "https://files.pythonhosted.org/packages/94/fa/4d09f814779d0ea80a28ed8e4c6662ec9a4a8ecef0ac52190ebac6262d14/jiter-0.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9100ddbec09741cc66feb0fc6773f8bdbd0e3c345689368f260082ff85dcc0cd", size = 393688 }, + { url = "https://files.pythonhosted.org/packages/54/9d/8eb5d4fb8bf7e93a75964a5da71a75c67c864baf7fa3f98598187b3c7e57/jiter-0.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ae1b0d82ac2d987f9ea512b1c9adfcc71a28de3dea3a6039b54d76cffda9901e", size = 520853 }, + { url = "https://files.pythonhosted.org/packages/e7/2c/5e07874e59e623a943a0acf1552a80d05b70f31b402287a8fc6d7ec634c7/jiter-0.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8020c99ec13a7db2b6f96cbe82ef4721c88b426a4892f27478044af0284615ef", size = 551016 }, + { url = "https://files.pythonhosted.org/packages/22/ed/d2d34422143474cadc15b60d482b1c35683dbc5c63c24346ddd0df09bcaf/jiter-0.15.0-cp311-cp311-win32.whl", hash = "sha256:42bfb257930800cf43e7c62c832402c704ab60797c992faf88d20e903eac8f32", size = 209518 }, + { url = "https://files.pythonhosted.org/packages/1d/7d/52778b930e5cc3e52a37d950b1c10494244308b4329b25a0ff0d88303a81/jiter-0.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:860a74063284a2ae9bfedd694f299cc2c68e2696c5f3d440cc9d18bb81b9dd04", size = 200565 }, + { url = "https://files.pythonhosted.org/packages/3b/4f/d9b4067feb69b3fa6eb0488e1b59e2ad5b463fe39f59e527eab2aca00bb0/jiter-0.15.0-cp311-cp311-win_arm64.whl", hash = "sha256:37a10c377ce3a4a85f4a67f28b7afe093154cde77eaf248a72e856aa08b4d865", size = 195488 }, + { url = "https://files.pythonhosted.org/packages/44/53/4f6bddbcde3c71e56d0aa1337ec95950f3d27dd4153e25aadf0feac71751/jiter-0.15.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:0e90a1c315a0226ec822d973817967f9223b7701546c8c2a7913e7ab0926294d", size = 308793 }, + { url = "https://files.pythonhosted.org/packages/01/84/c01099b59a285a1ebba64ae93f62bfa036675340fd1b0045ae65890a0442/jiter-0.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8c9004af7c8d67cce7f1aae1026fb55607f4aa600710d08ede3a3ce4aeefe7e0", size = 309570 }, + { url = "https://files.pythonhosted.org/packages/58/64/8fb7f9d45bb98190355454cd04dad8d8f27223d6bd52f83af07f637168a6/jiter-0.15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c210f8b35dc6f30aafd4b4365ca89b9d1189f21ab49b8e68fa6322a847aef138", size = 336783 }, + { url = "https://files.pythonhosted.org/packages/c3/b6/f5739011d009b3a30f6a53c5240979030ba29ae46a8c67e3a15759f7c37d/jiter-0.15.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f30bae8bc1c2d613e28e5af3e8cceb09b742f1c8a8a5f839fb67afaffc03b61", size = 363555 }, + { url = "https://files.pythonhosted.org/packages/e5/12/98a9d9f766665e8a3b6252454e17cb0c464606a28cf2fa09399b003345fa/jiter-0.15.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c60e71b6d10cfc284c9bf36bd885e8d44c46f688ce50aa91b5edd90181dea687", size = 452255 }, + { url = "https://files.pythonhosted.org/packages/e8/d5/60f972840f79c5e7544fce567c56f1e4e50468f996baba3e78d823dd62a6/jiter-0.15.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ab068bce62a45aa3e7367eceaffb5dde60b7eb853be8dece45132e3d0ff4879", size = 373559 }, + { url = "https://files.pythonhosted.org/packages/ee/cf/d46ef1234ba335aabc2f013210db8e0821a22f5e644a2e9449df199ecc23/jiter-0.15.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa248c9eb220197d363f688818dac2fd4b2f0cd7d843ca7105d652034823427d", size = 346055 }, + { url = "https://files.pythonhosted.org/packages/f0/63/4d2749d8d54d230bad9b3a6b0d00cc28c6ff6b2fdffc26a8ccf76cc5a974/jiter-0.15.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2a77aadd57cac1682e4401a72724d2796d89a4ba129b1a5812aa94ee480826eb", size = 351406 }, + { url = "https://files.pythonhosted.org/packages/d9/b9/9965b990035d8773328e0a8c8b457a87bf2b19f6c4126d9d99296be5d16a/jiter-0.15.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2ae901f3a55bfafdde31d289590fa25e3245735a2b1e8c7cc15871710a002871", size = 389357 }, + { url = "https://files.pythonhosted.org/packages/2d/55/9ddf903deda1413e87fed792f416b7123daee5b8efbad6a202a7421c36a5/jiter-0.15.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f0b271b462769543716f92d3a4f90527df6ef5ed05ee95ec4137f513e21e1b77", size = 517263 }, + { url = "https://files.pythonhosted.org/packages/e8/76/a0c40ad064d3a20a4fde231e35d56e9a01ce82164278180e82d5daf85469/jiter-0.15.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2fb6a5d26af81fc0f00f9360a891e05cf755e149bba391c4d563adc54812973d", size = 548646 }, + { url = "https://files.pythonhosted.org/packages/23/4f/eca9b954942916ba2f453891b8593ab444cd872396fe66a3936616f236f3/jiter-0.15.0-cp312-cp312-win32.whl", hash = "sha256:c2f6bb8b5216ab9e7873bc08b5d7bef2b8abbb578a3069bf1cd14a45d71d771d", size = 206427 }, + { url = "https://files.pythonhosted.org/packages/95/bf/8ead82a87495149542748e828d153fd232a512a22c83b02c4815c1a9c7d8/jiter-0.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:40b2c7e92c44a84d748d21706c68dc6ff8161d80b59c99d774721a0d2317d7c7", size = 197300 }, + { url = "https://files.pythonhosted.org/packages/f4/e4/9b8a78fb2d894471bc344e37f1949bdd784bd914d031dba0ba3a40c71dd7/jiter-0.15.0-cp312-cp312-win_arm64.whl", hash = "sha256:cc0bc345cf2df9d1c00ac443f50d543c1ccfa8b0422cb85b1ab70d681c0b255b", size = 192702 }, + { url = "https://files.pythonhosted.org/packages/e5/f4/f708c900ecee41b2025ef8413d5351e5649eb2125c506f6720cc69b06f5c/jiter-0.15.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:1c11465f97e2abf45a014b83b730222f8f1c5335e802c7055a67d50de6f1f4e3", size = 307829 }, + { url = "https://files.pythonhosted.org/packages/86/59/db537c0949e83668c38481d426b9f2fd5ab758c4ee53a811dd0a510626a0/jiter-0.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d1e7b1776f0797956c509e123d0952d10d293a9492dea9f288ab9570ec01d1a5", size = 308445 }, + { url = "https://files.pythonhosted.org/packages/37/38/ea0e13b18c30ef951da0d47d39e7fa9edb82a93a62990ffbd7cea9b622d4/jiter-0.15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:351a341c2105aa430b7047e30f1bf7975f6313b00165d3fc07be2edaf741f279", size = 336181 }, + { url = "https://files.pythonhosted.org/packages/58/fc/2303901b16c4ba05865588990a420c0b4156270b44379c20931544a1d962/jiter-0.15.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ab395feec8d249ec4044e228e98a7033f043426a265df439dc3698823f0a4e4", size = 362985 }, + { url = "https://files.pythonhosted.org/packages/5b/6f/11bace093c52e7d4d26c8e606ccd7ae8c972189622469ec0d9e28161e28b/jiter-0.15.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2a438005b6f22d0273413484d6094d7c2c5d10ec1b3a3bf128e0d1d3ba53258", size = 453292 }, + { url = "https://files.pythonhosted.org/packages/22/db/987f2f086ca4d7a6582eb4ccd513f9b26b42d9e4243a087609a3137a8fc7/jiter-0.15.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f18f85e4218d1b40f000f42a92239a7a61a902cd42c65e6c360dbd17dcb20894", size = 373501 }, + { url = "https://files.pythonhosted.org/packages/8f/7c/89fbcabb2739b7a5b8dc959a1b6c5761f6484f5fed3486854b3c789bb1de/jiter-0.15.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1aa62e277fc1cbd80e6deacae6f4d983b41b3d7728e0645c5d741a6149bba45", size = 344683 }, + { url = "https://files.pythonhosted.org/packages/30/6f/6cca7692e7dddfec6d8d76c54dc97f2af2a41df4ac0674b999df1f09a5f3/jiter-0.15.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:6550fa135c7deb8ead6af49ed7ff648532ea8334a1447fe34a36315ef79c5c29", size = 350892 }, + { url = "https://files.pythonhosted.org/packages/39/14/0338d6190cb8e6d22e677ab1d4eabd4117f67cca70c54cd04b82ff64e068/jiter-0.15.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:066f8f33f18b2419cd8213b2436fa7fbc9c499f315971cfa3ce1f9820c001b1b", size = 388723 }, + { url = "https://files.pythonhosted.org/packages/90/31/cc19f4a1bdb6afb09ce6a2f2615aa8d44d994eba0d8e6105ed1af920e736/jiter-0.15.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:75e8a04e91432dde9f1838373cf93d23726c79d3e908d319acf0e796f85592e7", size = 516648 }, + { url = "https://files.pythonhosted.org/packages/49/9f/833c541512cd091b63c10c0381973dfe11bc7a503a818c16384417e0c81e/jiter-0.15.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:a97261f1fccb8e50ecd2890a96e46efdc3f57c80a197324c6777827231eca712", size = 547382 }, + { url = "https://files.pythonhosted.org/packages/d2/11/e7b70e91f90bc4477e8eee9e8a5f7cf3cb41b4525d6394dc98a714eb8f7f/jiter-0.15.0-cp313-cp313-win32.whl", hash = "sha256:c77496cb10bd7549690fbbab3e5ec05857b83e49276f4a9423a766ddd2afcd4c", size = 205845 }, + { url = "https://files.pythonhosted.org/packages/4b/23/5c20d9ad6f02c493e4023e5d2d09e1c1f15fe2753c9102c544aff068a88e/jiter-0.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:b15741f501469009ae0ae90b7147958a664a7dede40aa7ff174a8a4645f546d0", size = 196842 }, + { url = "https://files.pythonhosted.org/packages/6b/11/1eb400ef248e8c925fd883fbe325daf5e42cd1b0d308539dd332bd4f7ffc/jiter-0.15.0-cp313-cp313-win_arm64.whl", hash = "sha256:5d6a60072b44c3c2b797a7ddcbcbbf2b34ea3cfd4721580fbfd2a09d9d9b84ba", size = 192212 }, + { url = "https://files.pythonhosted.org/packages/8a/60/2fd8d7c79da8acf9b7b277c7616847773779356b92acfc9bb158452174da/jiter-0.15.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:ef1fd24d9413f6209e00d3d5a453e67acfe004a25cc6c8e8484faed4311ab9e8", size = 315065 }, + { url = "https://files.pythonhosted.org/packages/46/f4/008fb7d65e8ac2abf00811651a661e025c4ba80bbc6f378450384ddd3aed/jiter-0.15.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:144f8e72cb53dab146347b91cceac01f5481237f2b93b4a339a1ee8f8878b67c", size = 339444 }, + { url = "https://files.pythonhosted.org/packages/00/55/90b0c7b9c6896c0f2a591dd36d36b71d22e09674bfef178fa03ba3f81499/jiter-0.15.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:553fcac2ef2cb990877f9fc0833b8b629a3e6a5670b6b5fd58219b41a653ddc4", size = 347779 }, + { url = "https://files.pythonhosted.org/packages/51/6b/69666cec5000fd57734c118437394516c749ae8dbeea9fb66d6fef9c4775/jiter-0.15.0-cp313-cp313t-win_amd64.whl", hash = "sha256:774f93f65031856bf14ad9f59bdcab8b8cad501e5ceabd51ba3525f76937a25b", size = 200395 }, + { url = "https://files.pythonhosted.org/packages/39/04/a6aa62cd27e8149b0d28df5561f10f6cceaf7935a9ccf3f1c5a05f9a0cd8/jiter-0.15.0-cp313-cp313t-win_arm64.whl", hash = "sha256:f1e1754960f38ec40613a07e5e372df67acb3b890fb383b6fb3de3e49ddbf3c7", size = 190516 }, + { url = "https://files.pythonhosted.org/packages/eb/d2/079f350ebf7859d081de30aa890f9e3be68516f754f3ba32366ffff4dcee/jiter-0.15.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:ac0d9ddea4350974be7a221fc25895f251a8fee748c889bdced2141c0fec1a49", size = 308884 }, + { url = "https://files.pythonhosted.org/packages/04/4e/a2c30a7f69b48c03b20935d647479106fe932f6e63f75faf53937197e05d/jiter-0.15.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:01a8222cf05ab1128e239421156c207949808acaaea2bdfd33130ae666786e86", size = 310028 }, + { url = "https://files.pythonhosted.org/packages/40/90/2e7cdfd3cf8ca967be38c48f5cf474d79f089efaf559a40f15984a77ae69/jiter-0.15.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:182226cbc930c9fab81bc2e41a4da672f89539906dadb05e75670ac07b94f71f", size = 337485 }, + { url = "https://files.pythonhosted.org/packages/9b/11/15a1aa28b120b8ee5b4f1fb894c125046225f09847738bd64233d3b84883/jiter-0.15.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:71683c38c825452999b5717fcae07ea708e8c93003e808be4319c1b02e3d176e", size = 364223 }, + { url = "https://files.pythonhosted.org/packages/b7/25/f442e8af5f3d0dcf47b39e83a0efd9ee45ea946aa6d04625dc3181eae3b6/jiter-0.15.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30f2218e6a9e5c18bc10fe6d41ac189c442c88eacf11bad9f28ef95a9bef00e6", size = 456387 }, + { url = "https://files.pythonhosted.org/packages/da/f4/37f2d2c9f64f49af7da652ed7532bb5a2372e588e6927c3fdd76f911db65/jiter-0.15.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5157de9f76eb4bc5ea74a1219366a25f945ad305641d74e04f59c54087091aa9", size = 374461 }, + { url = "https://files.pythonhosted.org/packages/60/28/edcfbbbf0cb15436f36664a8908a0df47ab9006298d4cd937dc08ea932d6/jiter-0.15.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90c5db5527c221249a876160663ab891ace358c17f7b9c93ec1478b7f0550e5c", size = 345924 }, + { url = "https://files.pythonhosted.org/packages/47/13/89fba6398dab7f202b7278c4b4aac122399d2c0183971c4a57a3b7088df5/jiter-0.15.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:3e4540b8e74e4268811ac05db226a6a128ff572e7e0ce3f1163b693cadb184cd", size = 352283 }, + { url = "https://files.pythonhosted.org/packages/1b/da/0f6af8cef2c565a1ab44d970f268c43ccaa72707386ea6388e6fe2b6cd26/jiter-0.15.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:62ebd14e47e9aed9df4472afcb2663668ce4d74891cd54f86bf6e44029d6dc89", size = 389985 }, + { url = "https://files.pythonhosted.org/packages/a1/ec/b9cb7d6d29e24ee14910266157d2a279d7a8f60ee0df7fa840882976ba64/jiter-0.15.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:0be6f5ad41a809f303f416d17cec92a7a725902fb9b4f3de3d19362ac0ef8554", size = 517695 }, + { url = "https://files.pythonhosted.org/packages/64/5e/6d1bda880723aae0ad86b4b763f044362448efe31e3e819635d41cb03451/jiter-0.15.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:813dfbb17d65328bf86e5f0905dd277ba2265d3ca20556e86c0c7035b7182e5a", size = 548868 }, + { url = "https://files.pythonhosted.org/packages/0c/72/7de501cf38dcacaf35098796f3a50e0f2e338baba18a58946c618544b809/jiter-0.15.0-cp314-cp314-win32.whl", hash = "sha256:50e51156192722a9c58db112837d3f8ef96fb3c5ecc14e95f409134b08b158ec", size = 206380 }, + { url = "https://files.pythonhosted.org/packages/1e/a9/e19addf4b0c1bdce52c6da12351e6bc42c340c45e7c09e2158e46d293ccc/jiter-0.15.0-cp314-cp314-win_amd64.whl", hash = "sha256:30ce1a5d16b5641dc935d50ef775af6a0871e3d14ab05d6fc54dff371b78e558", size = 197687 }, + { url = "https://files.pythonhosted.org/packages/f2/c9/776b1db01db25fc6c1d58d1979a37b0a9fe787e5f5b1d062d2eaacb77923/jiter-0.15.0-cp314-cp314-win_arm64.whl", hash = "sha256:510c8b3c17a0ed9ac69850c0438dada3c9b82d9c4d589fcb62002a5a9cf3a866", size = 192571 }, + { url = "https://files.pythonhosted.org/packages/a0/f6/45bb4670bacf300fd2c7abadbfb3af376e5f1b6ae75fd9bc069891d15870/jiter-0.15.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7553333dd0930c104a5a0db8df72bf7219fe663d731383b576bb6ed6351c984d", size = 317151 }, + { url = "https://files.pythonhosted.org/packages/d7/68/ed635ad5acd7b73e454283083bbb7c8205ad10e88b0d9d7d793b09fe8226/jiter-0.15.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2143ab06181d2b029eedcb6af3cebe95f11bbac62441781860f98ee9330a6a6", size = 341243 }, + { url = "https://files.pythonhosted.org/packages/5d/db/3ff4176b817b8ea33879e71e13d8bc2b0d481a7ed3fe9e080f333d415c16/jiter-0.15.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:6eac374c5c975709b69c10f09afd199df74150172156ad10c8d4fd785b7da995", size = 363629 }, + { url = "https://files.pythonhosted.org/packages/ab/24/5f8270e0ba9c883582f96f722f8a0b58015c7ce1f8c6d4571cf394e99b6b/jiter-0.15.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b3b3b775e33d3bfaec9899edc526ae97b0da0bf9d071a46124ba419149a414f8", size = 456198 }, + { url = "https://files.pythonhosted.org/packages/45/5b/76fc02b0b5c54c3d18c60653156e2f76fde1816f9b4722db68d6ee2c897e/jiter-0.15.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:eda3071db3346334beae1360b46da4606da57bf3528c167b3c38533afaf9f2c5", size = 373710 }, + { url = "https://files.pythonhosted.org/packages/c4/52/4310821b0ea9277994d3e1f49fc6a4b34e4800caebacb2c0af81da59a454/jiter-0.15.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c6694a173ecabc12eb60efbc0b474464ead1951ff65cd8b1e72100715c64512b", size = 349901 }, + { url = "https://files.pythonhosted.org/packages/93/fe/67648c35b3594fba8854ac64cc8a826d8bcd18324bbdb53d77697c60b6ef/jiter-0.15.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:a254e10b593624d230c365b6d616b22ca0ad65e63a16e6631c2b3466022e6ba8", size = 352438 }, + { url = "https://files.pythonhosted.org/packages/cb/28/0a1879d07ad6b3e025a2750027363452ced93c2d16d1c9d4b153ffd51c91/jiter-0.15.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d8d2955167274e15d79a7a020afdd9b39c990eb80b2d89fca695d92dcfdd38ec", size = 388152 }, + { url = "https://files.pythonhosted.org/packages/c1/78/46c6f6b56ba85c90021f4afd72ed42f691f8f84daacb5fe27277070e3858/jiter-0.15.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:acf4ee4d1fc55917239fe72972fb292dd773055d05eb040d36f4326e02cc2c0e", size = 517707 }, + { url = "https://files.pythonhosted.org/packages/ca/cb/720662d4c88fcad606e826fef5424365527ba43ce4868a479aed8f8c507e/jiter-0.15.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:e7196e56f1cd69af1dbb07dff02dcfb260a50b45a82d409d92a06fedb32473b5", size = 548241 }, + { url = "https://files.pythonhosted.org/packages/60/e3/935b8034fd143f21125c87d51404a9e0e1449186a494405721ff5d1d695e/jiter-0.15.0-cp314-cp314t-win32.whl", hash = "sha256:7f6163c0f10b055245f814dcc59f4818da60dfe72f3e72ab89fc24b6bd5e9c52", size = 207950 }, + { url = "https://files.pythonhosted.org/packages/93/59/984fd9ece895953dad3e0880a650e766f5a2da2c5514f0eafdaaabbeb5f9/jiter-0.15.0-cp314-cp314t-win_amd64.whl", hash = "sha256:980c256edb05b78a111b99c4de3b1d32e31634b867fd1fc2cf726e7b7bba9854", size = 200055 }, + { url = "https://files.pythonhosted.org/packages/0e/a4/cf8d779feb133a27a2e3bc833bccb9e13aa332cdf820497ebf72c10ce8c3/jiter-0.15.0-cp314-cp314t-win_arm64.whl", hash = "sha256:66b1880df2d01e206e8339769d1c7c1753bcb653efd6289e203f6f24ebada0c0", size = 191244 }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419 }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630 }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437 }, +] + +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687 }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/db/fefacb2136439fc8dd20e797950e749aa1f4997ed584c62cfb8ef7c2be0e/markupsafe-3.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad", size = 11631 }, + { url = "https://files.pythonhosted.org/packages/e1/2e/5898933336b61975ce9dc04decbc0a7f2fee78c30353c5efba7f2d6ff27a/markupsafe-3.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a", size = 12058 }, + { url = "https://files.pythonhosted.org/packages/1d/09/adf2df3699d87d1d8184038df46a9c80d78c0148492323f4693df54e17bb/markupsafe-3.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50", size = 24287 }, + { url = "https://files.pythonhosted.org/packages/30/ac/0273f6fcb5f42e314c6d8cd99effae6a5354604d461b8d392b5ec9530a54/markupsafe-3.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf", size = 22940 }, + { url = "https://files.pythonhosted.org/packages/19/ae/31c1be199ef767124c042c6c3e904da327a2f7f0cd63a0337e1eca2967a8/markupsafe-3.0.3-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f", size = 21887 }, + { url = "https://files.pythonhosted.org/packages/b2/76/7edcab99d5349a4532a459e1fe64f0b0467a3365056ae550d3bcf3f79e1e/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a", size = 23692 }, + { url = "https://files.pythonhosted.org/packages/a4/28/6e74cdd26d7514849143d69f0bf2399f929c37dc2b31e6829fd2045b2765/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115", size = 21471 }, + { url = "https://files.pythonhosted.org/packages/62/7e/a145f36a5c2945673e590850a6f8014318d5577ed7e5920a4b3448e0865d/markupsafe-3.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a", size = 22923 }, + { url = "https://files.pythonhosted.org/packages/0f/62/d9c46a7f5c9adbeeeda52f5b8d802e1094e9717705a645efc71b0913a0a8/markupsafe-3.0.3-cp311-cp311-win32.whl", hash = "sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19", size = 14572 }, + { url = "https://files.pythonhosted.org/packages/83/8a/4414c03d3f891739326e1783338e48fb49781cc915b2e0ee052aa490d586/markupsafe-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01", size = 15077 }, + { url = "https://files.pythonhosted.org/packages/35/73/893072b42e6862f319b5207adc9ae06070f095b358655f077f69a35601f0/markupsafe-3.0.3-cp311-cp311-win_arm64.whl", hash = "sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c", size = 13876 }, + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615 }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020 }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332 }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947 }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962 }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760 }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529 }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015 }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540 }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105 }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906 }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622 }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029 }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374 }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980 }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990 }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784 }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588 }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041 }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543 }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113 }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911 }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658 }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066 }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639 }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569 }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284 }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801 }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769 }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642 }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612 }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200 }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973 }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619 }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029 }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408 }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005 }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048 }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821 }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606 }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043 }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747 }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341 }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073 }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661 }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069 }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670 }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598 }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261 }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835 }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733 }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672 }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819 }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426 }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146 }, +] + +[[package]] +name = "mcp" +version = "1.27.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "jsonschema" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/83/d1efe7c2980d8a3afa476f4e3d42d53dd54c0ab94c27bee5d755b45c8b73/mcp-1.27.1.tar.gz", hash = "sha256:0f47e1820f8f8f941466b39749eb1d1839a04caddca2bc60e9d46e8a99914924", size = 608458 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/73/42d9596facebdb533b7f0b86c1b0364ef350d1f8ba78b1052e8a58b48b65/mcp-1.27.1-py3-none-any.whl", hash = "sha256:1af3c4203b329430fde7a87b4fcb6392a041f5cb851fd68fc674016ab4e7c06f", size = 216260 }, +] + +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979 }, +] + +[[package]] +name = "multidict" +version = "6.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/c2/c2d94cbe6ac1753f3fc980da97b3d930efe1da3af3c9f5125354436c073d/multidict-6.7.1.tar.gz", hash = "sha256:ec6652a1bee61c53a3e5776b6049172c53b6aaba34f18c9ad04f82712bac623d", size = 102010 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/f1/a90635c4f88fb913fbf4ce660b83b7445b7a02615bda034b2f8eb38fd597/multidict-6.7.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7ff981b266af91d7b4b3793ca3382e53229088d193a85dfad6f5f4c27fc73e5d", size = 76626 }, + { url = "https://files.pythonhosted.org/packages/a6/9b/267e64eaf6fc637a15b35f5de31a566634a2740f97d8d094a69d34f524a4/multidict-6.7.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:844c5bca0b5444adb44a623fb0a1310c2f4cd41f402126bb269cd44c9b3f3e1e", size = 44706 }, + { url = "https://files.pythonhosted.org/packages/dd/a4/d45caf2b97b035c57267791ecfaafbd59c68212004b3842830954bb4b02e/multidict-6.7.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f2a0a924d4c2e9afcd7ec64f9de35fcd96915149b2216e1cb2c10a56df483855", size = 44356 }, + { url = "https://files.pythonhosted.org/packages/fd/d2/0a36c8473f0cbaeadd5db6c8b72d15bbceeec275807772bfcd059bef487d/multidict-6.7.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8be1802715a8e892c784c0197c2ace276ea52702a0ede98b6310c8f255a5afb3", size = 244355 }, + { url = "https://files.pythonhosted.org/packages/5d/16/8c65be997fd7dd311b7d39c7b6e71a0cb449bad093761481eccbbe4b42a2/multidict-6.7.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2e2d2ed645ea29f31c4c7ea1552fcfd7cb7ba656e1eafd4134a6620c9f5fdd9e", size = 246433 }, + { url = "https://files.pythonhosted.org/packages/01/fb/4dbd7e848d2799c6a026ec88ad39cf2b8416aa167fcc903baa55ecaa045c/multidict-6.7.1-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:95922cee9a778659e91db6497596435777bd25ed116701a4c034f8e46544955a", size = 225376 }, + { url = "https://files.pythonhosted.org/packages/b6/8a/4a3a6341eac3830f6053062f8fbc9a9e54407c80755b3f05bc427295c2d0/multidict-6.7.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6b83cabdc375ffaaa15edd97eb7c0c672ad788e2687004990074d7d6c9b140c8", size = 257365 }, + { url = "https://files.pythonhosted.org/packages/f7/a2/dd575a69c1aa206e12d27d0770cdf9b92434b48a9ef0cd0d1afdecaa93c4/multidict-6.7.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:38fb49540705369bab8484db0689d86c0a33a0a9f2c1b197f506b71b4b6c19b0", size = 254747 }, + { url = "https://files.pythonhosted.org/packages/5a/56/21b27c560c13822ed93133f08aa6372c53a8e067f11fbed37b4adcdac922/multidict-6.7.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:439cbebd499f92e9aa6793016a8acaa161dfa749ae86d20960189f5398a19144", size = 246293 }, + { url = "https://files.pythonhosted.org/packages/5a/a4/23466059dc3854763423d0ad6c0f3683a379d97673b1b89ec33826e46728/multidict-6.7.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d3bc717b6fe763b8be3f2bee2701d3c8eb1b2a8ae9f60910f1b2860c82b6c49", size = 242962 }, + { url = "https://files.pythonhosted.org/packages/1f/67/51dd754a3524d685958001e8fa20a0f5f90a6a856e0a9dcabff69be3dbb7/multidict-6.7.1-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:619e5a1ac57986dbfec9f0b301d865dddf763696435e2962f6d9cf2fdff2bb71", size = 237360 }, + { url = "https://files.pythonhosted.org/packages/64/3f/036dfc8c174934d4b55d86ff4f978e558b0e585cef70cfc1ad01adc6bf18/multidict-6.7.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0b38ebffd9be37c1170d33bc0f36f4f262e0a09bc1aac1c34c7aa51a7293f0b3", size = 245940 }, + { url = "https://files.pythonhosted.org/packages/3d/20/6214d3c105928ebc353a1c644a6ef1408bc5794fcb4f170bb524a3c16311/multidict-6.7.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:10ae39c9cfe6adedcdb764f5e8411d4a92b055e35573a2eaa88d3323289ef93c", size = 253502 }, + { url = "https://files.pythonhosted.org/packages/b1/e2/c653bc4ae1be70a0f836b82172d643fcf1dade042ba2676ab08ec08bff0f/multidict-6.7.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:25167cc263257660290fba06b9318d2026e3c910be240a146e1f66dd114af2b0", size = 247065 }, + { url = "https://files.pythonhosted.org/packages/c8/11/a854b4154cd3bd8b1fd375e8a8ca9d73be37610c361543d56f764109509b/multidict-6.7.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:128441d052254f42989ef98b7b6a6ecb1e6f708aa962c7984235316db59f50fa", size = 241870 }, + { url = "https://files.pythonhosted.org/packages/13/bf/9676c0392309b5fdae322333d22a829715b570edb9baa8016a517b55b558/multidict-6.7.1-cp311-cp311-win32.whl", hash = "sha256:d62b7f64ffde3b99d06b707a280db04fb3855b55f5a06df387236051d0668f4a", size = 41302 }, + { url = "https://files.pythonhosted.org/packages/c9/68/f16a3a8ba6f7b6dc92a1f19669c0810bd2c43fc5a02da13b1cbf8e253845/multidict-6.7.1-cp311-cp311-win_amd64.whl", hash = "sha256:bdbf9f3b332abd0cdb306e7c2113818ab1e922dc84b8f8fd06ec89ed2a19ab8b", size = 45981 }, + { url = "https://files.pythonhosted.org/packages/ac/ad/9dd5305253fa00cd3c7555dbef69d5bf4133debc53b87ab8d6a44d411665/multidict-6.7.1-cp311-cp311-win_arm64.whl", hash = "sha256:b8c990b037d2fff2f4e33d3f21b9b531c5745b33a49a7d6dbe7a177266af44f6", size = 43159 }, + { url = "https://files.pythonhosted.org/packages/8d/9c/f20e0e2cf80e4b2e4b1c365bf5fe104ee633c751a724246262db8f1a0b13/multidict-6.7.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a90f75c956e32891a4eda3639ce6dd86e87105271f43d43442a3aedf3cddf172", size = 76893 }, + { url = "https://files.pythonhosted.org/packages/fe/cf/18ef143a81610136d3da8193da9d80bfe1cb548a1e2d1c775f26b23d024a/multidict-6.7.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3fccb473e87eaa1382689053e4a4618e7ba7b9b9b8d6adf2027ee474597128cd", size = 45456 }, + { url = "https://files.pythonhosted.org/packages/a9/65/1caac9d4cd32e8433908683446eebc953e82d22b03d10d41a5f0fefe991b/multidict-6.7.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b0fa96985700739c4c7853a43c0b3e169360d6855780021bfc6d0f1ce7c123e7", size = 43872 }, + { url = "https://files.pythonhosted.org/packages/cf/3b/d6bd75dc4f3ff7c73766e04e705b00ed6dbbaccf670d9e05a12b006f5a21/multidict-6.7.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cb2a55f408c3043e42b40cc8eecd575afa27b7e0b956dfb190de0f8499a57a53", size = 251018 }, + { url = "https://files.pythonhosted.org/packages/fd/80/c959c5933adedb9ac15152e4067c702a808ea183a8b64cf8f31af8ad3155/multidict-6.7.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb0ce7b2a32d09892b3dd6cc44877a0d02a33241fafca5f25c8b6b62374f8b75", size = 258883 }, + { url = "https://files.pythonhosted.org/packages/86/85/7ed40adafea3d4f1c8b916e3b5cc3a8e07dfcdcb9cd72800f4ed3ca1b387/multidict-6.7.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c3a32d23520ee37bf327d1e1a656fec76a2edd5c038bf43eddfa0572ec49c60b", size = 242413 }, + { url = "https://files.pythonhosted.org/packages/d2/57/b8565ff533e48595503c785f8361ff9a4fde4d67de25c207cd0ba3befd03/multidict-6.7.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9c90fed18bffc0189ba814749fdcc102b536e83a9f738a9003e569acd540a733", size = 268404 }, + { url = "https://files.pythonhosted.org/packages/e0/50/9810c5c29350f7258180dfdcb2e52783a0632862eb334c4896ac717cebcb/multidict-6.7.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:da62917e6076f512daccfbbde27f46fed1c98fee202f0559adec8ee0de67f71a", size = 269456 }, + { url = "https://files.pythonhosted.org/packages/f3/8d/5e5be3ced1d12966fefb5c4ea3b2a5b480afcea36406559442c6e31d4a48/multidict-6.7.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bfde23ef6ed9db7eaee6c37dcec08524cb43903c60b285b172b6c094711b3961", size = 256322 }, + { url = "https://files.pythonhosted.org/packages/31/6e/d8a26d81ac166a5592782d208dd90dfdc0a7a218adaa52b45a672b46c122/multidict-6.7.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3758692429e4e32f1ba0df23219cd0b4fc0a52f476726fff9337d1a57676a582", size = 253955 }, + { url = "https://files.pythonhosted.org/packages/59/4c/7c672c8aad41534ba619bcd4ade7a0dc87ed6b8b5c06149b85d3dd03f0cd/multidict-6.7.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:398c1478926eca669f2fd6a5856b6de9c0acf23a2cb59a14c0ba5844fa38077e", size = 251254 }, + { url = "https://files.pythonhosted.org/packages/7b/bd/84c24de512cbafbdbc39439f74e967f19570ce7924e3007174a29c348916/multidict-6.7.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c102791b1c4f3ab36ce4101154549105a53dc828f016356b3e3bcae2e3a039d3", size = 252059 }, + { url = "https://files.pythonhosted.org/packages/fa/ba/f5449385510825b73d01c2d4087bf6d2fccc20a2d42ac34df93191d3dd03/multidict-6.7.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a088b62bd733e2ad12c50dad01b7d0166c30287c166e137433d3b410add807a6", size = 263588 }, + { url = "https://files.pythonhosted.org/packages/d7/11/afc7c677f68f75c84a69fe37184f0f82fce13ce4b92f49f3db280b7e92b3/multidict-6.7.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:3d51ff4785d58d3f6c91bdbffcb5e1f7ddfda557727043aa20d20ec4f65e324a", size = 259642 }, + { url = "https://files.pythonhosted.org/packages/2b/17/ebb9644da78c4ab36403739e0e6e0e30ebb135b9caf3440825001a0bddcb/multidict-6.7.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc5907494fccf3e7d3f94f95c91d6336b092b5fc83811720fae5e2765890dfba", size = 251377 }, + { url = "https://files.pythonhosted.org/packages/ca/a4/840f5b97339e27846c46307f2530a2805d9d537d8b8bd416af031cad7fa0/multidict-6.7.1-cp312-cp312-win32.whl", hash = "sha256:28ca5ce2fd9716631133d0e9a9b9a745ad7f60bac2bccafb56aa380fc0b6c511", size = 41887 }, + { url = "https://files.pythonhosted.org/packages/80/31/0b2517913687895f5904325c2069d6a3b78f66cc641a86a2baf75a05dcbb/multidict-6.7.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcee94dfbd638784645b066074b338bc9cc155d4b4bffa4adce1615c5a426c19", size = 46053 }, + { url = "https://files.pythonhosted.org/packages/0c/5b/aba28e4ee4006ae4c7df8d327d31025d760ffa992ea23812a601d226e682/multidict-6.7.1-cp312-cp312-win_arm64.whl", hash = "sha256:ba0a9fb644d0c1a2194cf7ffb043bd852cea63a57f66fbd33959f7dae18517bf", size = 43307 }, + { url = "https://files.pythonhosted.org/packages/f2/22/929c141d6c0dba87d3e1d38fbdf1ba8baba86b7776469f2bc2d3227a1e67/multidict-6.7.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:2b41f5fed0ed563624f1c17630cb9941cf2309d4df00e494b551b5f3e3d67a23", size = 76174 }, + { url = "https://files.pythonhosted.org/packages/c7/75/bc704ae15fee974f8fccd871305e254754167dce5f9e42d88a2def741a1d/multidict-6.7.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:84e61e3af5463c19b67ced91f6c634effb89ef8bfc5ca0267f954451ed4bb6a2", size = 45116 }, + { url = "https://files.pythonhosted.org/packages/79/76/55cd7186f498ed080a18440c9013011eb548f77ae1b297206d030eb1180a/multidict-6.7.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:935434b9853c7c112eee7ac891bc4cb86455aa631269ae35442cb316790c1445", size = 43524 }, + { url = "https://files.pythonhosted.org/packages/e9/3c/414842ef8d5a1628d68edee29ba0e5bcf235dbfb3ccd3ea303a7fe8c72ff/multidict-6.7.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:432feb25a1cb67fe82a9680b4d65fb542e4635cb3166cd9c01560651ad60f177", size = 249368 }, + { url = "https://files.pythonhosted.org/packages/f6/32/befed7f74c458b4a525e60519fe8d87eef72bb1e99924fa2b0f9d97a221e/multidict-6.7.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e82d14e3c948952a1a85503817e038cba5905a3352de76b9a465075d072fba23", size = 256952 }, + { url = "https://files.pythonhosted.org/packages/03/d6/c878a44ba877f366630c860fdf74bfb203c33778f12b6ac274936853c451/multidict-6.7.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:4cfb48c6ea66c83bcaaf7e4dfa7ec1b6bbcf751b7db85a328902796dfde4c060", size = 240317 }, + { url = "https://files.pythonhosted.org/packages/68/49/57421b4d7ad2e9e60e25922b08ceb37e077b90444bde6ead629095327a6f/multidict-6.7.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1d540e51b7e8e170174555edecddbd5538105443754539193e3e1061864d444d", size = 267132 }, + { url = "https://files.pythonhosted.org/packages/b7/fe/ec0edd52ddbcea2a2e89e174f0206444a61440b40f39704e64dc807a70bd/multidict-6.7.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:273d23f4b40f3dce4d6c8a821c741a86dec62cded82e1175ba3d99be128147ed", size = 268140 }, + { url = "https://files.pythonhosted.org/packages/b0/73/6e1b01cbeb458807aa0831742232dbdd1fa92bfa33f52a3f176b4ff3dc11/multidict-6.7.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d624335fd4fa1c08a53f8b4be7676ebde19cd092b3895c421045ca87895b429", size = 254277 }, + { url = "https://files.pythonhosted.org/packages/6a/b2/5fb8c124d7561a4974c342bc8c778b471ebbeb3cc17df696f034a7e9afe7/multidict-6.7.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:12fad252f8b267cc75b66e8fc51b3079604e8d43a75428ffe193cd9e2195dfd6", size = 252291 }, + { url = "https://files.pythonhosted.org/packages/5a/96/51d4e4e06bcce92577fcd488e22600bd38e4fd59c20cb49434d054903bd2/multidict-6.7.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:03ede2a6ffbe8ef936b92cb4529f27f42be7f56afcdab5ab739cd5f27fb1cbf9", size = 250156 }, + { url = "https://files.pythonhosted.org/packages/db/6b/420e173eec5fba721a50e2a9f89eda89d9c98fded1124f8d5c675f7a0c0f/multidict-6.7.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:90efbcf47dbe33dcf643a1e400d67d59abeac5db07dc3f27d6bdeae497a2198c", size = 249742 }, + { url = "https://files.pythonhosted.org/packages/44/a3/ec5b5bd98f306bc2aa297b8c6f11a46714a56b1e6ef5ebda50a4f5d7c5fb/multidict-6.7.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c4b9bfc148f5a91be9244d6264c53035c8a0dcd2f51f1c3c6e30e30ebaa1c84", size = 262221 }, + { url = "https://files.pythonhosted.org/packages/cd/f7/e8c0d0da0cd1e28d10e624604e1a36bcc3353aaebdfdc3a43c72bc683a12/multidict-6.7.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:401c5a650f3add2472d1d288c26deebc540f99e2fb83e9525007a74cd2116f1d", size = 258664 }, + { url = "https://files.pythonhosted.org/packages/52/da/151a44e8016dd33feed44f730bd856a66257c1ee7aed4f44b649fb7edeb3/multidict-6.7.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:97891f3b1b3ffbded884e2916cacf3c6fc87b66bb0dde46f7357404750559f33", size = 249490 }, + { url = "https://files.pythonhosted.org/packages/87/af/a3b86bf9630b732897f6fc3f4c4714b90aa4361983ccbdcd6c0339b21b0c/multidict-6.7.1-cp313-cp313-win32.whl", hash = "sha256:e1c5988359516095535c4301af38d8a8838534158f649c05dd1050222321bcb3", size = 41695 }, + { url = "https://files.pythonhosted.org/packages/b2/35/e994121b0e90e46134673422dd564623f93304614f5d11886b1b3e06f503/multidict-6.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:960c83bf01a95b12b08fd54324a4eb1d5b52c88932b5cba5d6e712bb3ed12eb5", size = 45884 }, + { url = "https://files.pythonhosted.org/packages/ca/61/42d3e5dbf661242a69c97ea363f2d7b46c567da8eadef8890022be6e2ab0/multidict-6.7.1-cp313-cp313-win_arm64.whl", hash = "sha256:563fe25c678aaba333d5399408f5ec3c383ca5b663e7f774dd179a520b8144df", size = 43122 }, + { url = "https://files.pythonhosted.org/packages/6d/b3/e6b21c6c4f314bb956016b0b3ef2162590a529b84cb831c257519e7fde44/multidict-6.7.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:c76c4bec1538375dad9d452d246ca5368ad6e1c9039dadcf007ae59c70619ea1", size = 83175 }, + { url = "https://files.pythonhosted.org/packages/fb/76/23ecd2abfe0957b234f6c960f4ade497f55f2c16aeb684d4ecdbf1c95791/multidict-6.7.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:57b46b24b5d5ebcc978da4ec23a819a9402b4228b8a90d9c656422b4bdd8a963", size = 48460 }, + { url = "https://files.pythonhosted.org/packages/c4/57/a0ed92b23f3a042c36bc4227b72b97eca803f5f1801c1ab77c8a212d455e/multidict-6.7.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e954b24433c768ce78ab7929e84ccf3422e46deb45a4dc9f93438f8217fa2d34", size = 46930 }, + { url = "https://files.pythonhosted.org/packages/b5/66/02ec7ace29162e447f6382c495dc95826bf931d3818799bbef11e8f7df1a/multidict-6.7.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3bd231490fa7217cc832528e1cd8752a96f0125ddd2b5749390f7c3ec8721b65", size = 242582 }, + { url = "https://files.pythonhosted.org/packages/58/18/64f5a795e7677670e872673aca234162514696274597b3708b2c0d276cce/multidict-6.7.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:253282d70d67885a15c8a7716f3a73edf2d635793ceda8173b9ecc21f2fb8292", size = 250031 }, + { url = "https://files.pythonhosted.org/packages/c8/ed/e192291dbbe51a8290c5686f482084d31bcd9d09af24f63358c3d42fd284/multidict-6.7.1-cp313-cp313t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0b4c48648d7649c9335cf1927a8b87fa692de3dcb15faa676c6a6f1f1aabda43", size = 228596 }, + { url = "https://files.pythonhosted.org/packages/1e/7e/3562a15a60cf747397e7f2180b0a11dc0c38d9175a650e75fa1b4d325e15/multidict-6.7.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:98bc624954ec4d2c7cb074b8eefc2b5d0ce7d482e410df446414355d158fe4ca", size = 257492 }, + { url = "https://files.pythonhosted.org/packages/24/02/7d0f9eae92b5249bb50ac1595b295f10e263dd0078ebb55115c31e0eaccd/multidict-6.7.1-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1b99af4d9eec0b49927b4402bcbb58dea89d3e0db8806a4086117019939ad3dd", size = 255899 }, + { url = "https://files.pythonhosted.org/packages/00/e3/9b60ed9e23e64c73a5cde95269ef1330678e9c6e34dd4eb6b431b85b5a10/multidict-6.7.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6aac4f16b472d5b7dc6f66a0d49dd57b0e0902090be16594dc9ebfd3d17c47e7", size = 247970 }, + { url = "https://files.pythonhosted.org/packages/3e/06/538e58a63ed5cfb0bd4517e346b91da32fde409d839720f664e9a4ae4f9d/multidict-6.7.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:21f830fe223215dffd51f538e78c172ed7c7f60c9b96a2bf05c4848ad49921c3", size = 245060 }, + { url = "https://files.pythonhosted.org/packages/b2/2f/d743a3045a97c895d401e9bd29aaa09b94f5cbdf1bd561609e5a6c431c70/multidict-6.7.1-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:f5dd81c45b05518b9aa4da4aa74e1c93d715efa234fd3e8a179df611cc85e5f4", size = 235888 }, + { url = "https://files.pythonhosted.org/packages/38/83/5a325cac191ab28b63c52f14f1131f3b0a55ba3b9aa65a6d0bf2a9b921a0/multidict-6.7.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:eb304767bca2bb92fb9c5bd33cedc95baee5bb5f6c88e63706533a1c06ad08c8", size = 243554 }, + { url = "https://files.pythonhosted.org/packages/20/1f/9d2327086bd15da2725ef6aae624208e2ef828ed99892b17f60c344e57ed/multidict-6.7.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:c9035dde0f916702850ef66460bc4239d89d08df4d02023a5926e7446724212c", size = 252341 }, + { url = "https://files.pythonhosted.org/packages/e8/2c/2a1aa0280cf579d0f6eed8ee5211c4f1730bd7e06c636ba2ee6aafda302e/multidict-6.7.1-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:af959b9beeb66c822380f222f0e0a1889331597e81f1ded7f374f3ecb0fd6c52", size = 246391 }, + { url = "https://files.pythonhosted.org/packages/e5/03/7ca022ffc36c5a3f6e03b179a5ceb829be9da5783e6fe395f347c0794680/multidict-6.7.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:41f2952231456154ee479651491e94118229844dd7226541788be783be2b5108", size = 243422 }, + { url = "https://files.pythonhosted.org/packages/dc/1d/b31650eab6c5778aceed46ba735bd97f7c7d2f54b319fa916c0f96e7805b/multidict-6.7.1-cp313-cp313t-win32.whl", hash = "sha256:df9f19c28adcb40b6aae30bbaa1478c389efd50c28d541d76760199fc1037c32", size = 47770 }, + { url = "https://files.pythonhosted.org/packages/ac/5b/2d2d1d522e51285bd61b1e20df8f47ae1a9d80839db0b24ea783b3832832/multidict-6.7.1-cp313-cp313t-win_amd64.whl", hash = "sha256:d54ecf9f301853f2c5e802da559604b3e95bb7a3b01a9c295c6ee591b9882de8", size = 53109 }, + { url = "https://files.pythonhosted.org/packages/3d/a3/cc409ba012c83ca024a308516703cf339bdc4b696195644a7215a5164a24/multidict-6.7.1-cp313-cp313t-win_arm64.whl", hash = "sha256:5a37ca18e360377cfda1d62f5f382ff41f2b8c4ccb329ed974cc2e1643440118", size = 45573 }, + { url = "https://files.pythonhosted.org/packages/91/cc/db74228a8be41884a567e88a62fd589a913708fcf180d029898c17a9a371/multidict-6.7.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8f333ec9c5eb1b7105e3b84b53141e66ca05a19a605368c55450b6ba208cb9ee", size = 75190 }, + { url = "https://files.pythonhosted.org/packages/d5/22/492f2246bb5b534abd44804292e81eeaf835388901f0c574bac4eeec73c5/multidict-6.7.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a407f13c188f804c759fc6a9f88286a565c242a76b27626594c133b82883b5c2", size = 44486 }, + { url = "https://files.pythonhosted.org/packages/f1/4f/733c48f270565d78b4544f2baddc2fb2a245e5a8640254b12c36ac7ac68e/multidict-6.7.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0e161ddf326db5577c3a4cc2d8648f81456e8a20d40415541587a71620d7a7d1", size = 43219 }, + { url = "https://files.pythonhosted.org/packages/24/bb/2c0c2287963f4259c85e8bcbba9182ced8d7fca65c780c38e99e61629d11/multidict-6.7.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:1e3a8bb24342a8201d178c3b4984c26ba81a577c80d4d525727427460a50c22d", size = 245132 }, + { url = "https://files.pythonhosted.org/packages/a7/f9/44d4b3064c65079d2467888794dea218d1601898ac50222ab8a9a8094460/multidict-6.7.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:97231140a50f5d447d3164f994b86a0bed7cd016e2682f8650d6a9158e14fd31", size = 252420 }, + { url = "https://files.pythonhosted.org/packages/8b/13/78f7275e73fa17b24c9a51b0bd9d73ba64bb32d0ed51b02a746eb876abe7/multidict-6.7.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b10359683bd8806a200fd2909e7c8ca3a7b24ec1d8132e483d58e791d881048", size = 233510 }, + { url = "https://files.pythonhosted.org/packages/4b/25/8167187f62ae3cbd52da7893f58cb036b47ea3fb67138787c76800158982/multidict-6.7.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:283ddac99f7ac25a4acadbf004cb5ae34480bbeb063520f70ce397b281859362", size = 264094 }, + { url = "https://files.pythonhosted.org/packages/a1/e7/69a3a83b7b030cf283fb06ce074a05a02322359783424d7edf0f15fe5022/multidict-6.7.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:538cec1e18c067d0e6103aa9a74f9e832904c957adc260e61cd9d8cf0c3b3d37", size = 260786 }, + { url = "https://files.pythonhosted.org/packages/fe/3b/8ec5074bcfc450fe84273713b4b0a0dd47c0249358f5d82eb8104ffe2520/multidict-6.7.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7eee46ccb30ff48a1e35bb818cc90846c6be2b68240e42a78599166722cea709", size = 248483 }, + { url = "https://files.pythonhosted.org/packages/48/5a/d5a99e3acbca0e29c5d9cba8f92ceb15dce78bab963b308ae692981e3a5d/multidict-6.7.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fa263a02f4f2dd2d11a7b1bb4362aa7cb1049f84a9235d31adf63f30143469a0", size = 248403 }, + { url = "https://files.pythonhosted.org/packages/35/48/e58cd31f6c7d5102f2a4bf89f96b9cf7e00b6c6f3d04ecc44417c00a5a3c/multidict-6.7.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2e1425e2f99ec5bd36c15a01b690a1a2456209c5deed58f95469ffb46039ccbb", size = 240315 }, + { url = "https://files.pythonhosted.org/packages/94/33/1cd210229559cb90b6786c30676bb0c58249ff42f942765f88793b41fdce/multidict-6.7.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:497394b3239fc6f0e13a78a3e1b61296e72bf1c5f94b4c4eb80b265c37a131cd", size = 245528 }, + { url = "https://files.pythonhosted.org/packages/64/f2/6e1107d226278c876c783056b7db43d800bb64c6131cec9c8dfb6903698e/multidict-6.7.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:233b398c29d3f1b9676b4b6f75c518a06fcb2ea0b925119fb2c1bc35c05e1601", size = 258784 }, + { url = "https://files.pythonhosted.org/packages/4d/c1/11f664f14d525e4a1b5327a82d4de61a1db604ab34c6603bb3c2cc63ad34/multidict-6.7.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:93b1818e4a6e0930454f0f2af7dfce69307ca03cdcfb3739bf4d91241967b6c1", size = 251980 }, + { url = "https://files.pythonhosted.org/packages/e1/9f/75a9ac888121d0c5bbd4ecf4eead45668b1766f6baabfb3b7f66a410e231/multidict-6.7.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f33dc2a3abe9249ea5d8360f969ec7f4142e7ac45ee7014d8f8d5acddf178b7b", size = 243602 }, + { url = "https://files.pythonhosted.org/packages/9a/e7/50bf7b004cc8525d80dbbbedfdc7aed3e4c323810890be4413e589074032/multidict-6.7.1-cp314-cp314-win32.whl", hash = "sha256:3ab8b9d8b75aef9df299595d5388b14530839f6422333357af1339443cff777d", size = 40930 }, + { url = "https://files.pythonhosted.org/packages/e0/bf/52f25716bbe93745595800f36fb17b73711f14da59ed0bb2eba141bc9f0f/multidict-6.7.1-cp314-cp314-win_amd64.whl", hash = "sha256:5e01429a929600e7dab7b166062d9bb54a5eed752384c7384c968c2afab8f50f", size = 45074 }, + { url = "https://files.pythonhosted.org/packages/97/ab/22803b03285fa3a525f48217963da3a65ae40f6a1b6f6cf2768879e208f9/multidict-6.7.1-cp314-cp314-win_arm64.whl", hash = "sha256:4885cb0e817aef5d00a2e8451d4665c1808378dc27c2705f1bf4ef8505c0d2e5", size = 42471 }, + { url = "https://files.pythonhosted.org/packages/e0/6d/f9293baa6146ba9507e360ea0292b6422b016907c393e2f63fc40ab7b7b5/multidict-6.7.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:0458c978acd8e6ea53c81eefaddbbee9c6c5e591f41b3f5e8e194780fe026581", size = 82401 }, + { url = "https://files.pythonhosted.org/packages/7a/68/53b5494738d83558d87c3c71a486504d8373421c3e0dbb6d0db48ad42ee0/multidict-6.7.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c0abd12629b0af3cf590982c0b413b1e7395cd4ec026f30986818ab95bfaa94a", size = 48143 }, + { url = "https://files.pythonhosted.org/packages/37/e8/5284c53310dcdc99ce5d66563f6e5773531a9b9fe9ec7a615e9bc306b05f/multidict-6.7.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:14525a5f61d7d0c94b368a42cff4c9a4e7ba2d52e2672a7b23d84dc86fb02b0c", size = 46507 }, + { url = "https://files.pythonhosted.org/packages/e4/fc/6800d0e5b3875568b4083ecf5f310dcf91d86d52573160834fb4bfcf5e4f/multidict-6.7.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:17307b22c217b4cf05033dabefe68255a534d637c6c9b0cc8382718f87be4262", size = 239358 }, + { url = "https://files.pythonhosted.org/packages/41/75/4ad0973179361cdf3a113905e6e088173198349131be2b390f9fa4da5fc6/multidict-6.7.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a7e590ff876a3eaf1c02a4dfe0724b6e69a9e9de6d8f556816f29c496046e59", size = 246884 }, + { url = "https://files.pythonhosted.org/packages/c3/9c/095bb28b5da139bd41fb9a5d5caff412584f377914bd8787c2aa98717130/multidict-6.7.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:5fa6a95dfee63893d80a34758cd0e0c118a30b8dcb46372bf75106c591b77889", size = 225878 }, + { url = "https://files.pythonhosted.org/packages/07/d0/c0a72000243756e8f5a277b6b514fa005f2c73d481b7d9e47cd4568aa2e4/multidict-6.7.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a0543217a6a017692aa6ae5cc39adb75e587af0f3a82288b1492eb73dd6cc2a4", size = 253542 }, + { url = "https://files.pythonhosted.org/packages/c0/6b/f69da15289e384ecf2a68837ec8b5ad8c33e973aa18b266f50fe55f24b8c/multidict-6.7.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f99fe611c312b3c1c0ace793f92464d8cd263cc3b26b5721950d977b006b6c4d", size = 252403 }, + { url = "https://files.pythonhosted.org/packages/a2/76/b9669547afa5a1a25cd93eaca91c0da1c095b06b6d2d8ec25b713588d3a1/multidict-6.7.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9004d8386d133b7e6135679424c91b0b854d2d164af6ea3f289f8f2761064609", size = 244889 }, + { url = "https://files.pythonhosted.org/packages/7e/a9/a50d2669e506dad33cfc45b5d574a205587b7b8a5f426f2fbb2e90882588/multidict-6.7.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e628ef0e6859ffd8273c69412a2465c4be4a9517d07261b33334b5ec6f3c7489", size = 241982 }, + { url = "https://files.pythonhosted.org/packages/c5/bb/1609558ad8b456b4827d3c5a5b775c93b87878fd3117ed3db3423dfbce1b/multidict-6.7.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:841189848ba629c3552035a6a7f5bf3b02eb304e9fea7492ca220a8eda6b0e5c", size = 232415 }, + { url = "https://files.pythonhosted.org/packages/d8/59/6f61039d2aa9261871e03ab9dc058a550d240f25859b05b67fd70f80d4b3/multidict-6.7.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:ce1bbd7d780bb5a0da032e095c951f7014d6b0a205f8318308140f1a6aba159e", size = 240337 }, + { url = "https://files.pythonhosted.org/packages/a1/29/fdc6a43c203890dc2ae9249971ecd0c41deaedfe00d25cb6564b2edd99eb/multidict-6.7.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:b26684587228afed0d50cf804cc71062cc9c1cdf55051c4c6345d372947b268c", size = 248788 }, + { url = "https://files.pythonhosted.org/packages/a9/14/a153a06101323e4cf086ecee3faadba52ff71633d471f9685c42e3736163/multidict-6.7.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9f9af11306994335398293f9958071019e3ab95e9a707dc1383a35613f6abcb9", size = 242842 }, + { url = "https://files.pythonhosted.org/packages/41/5f/604ae839e64a4a6efc80db94465348d3b328ee955e37acb24badbcd24d83/multidict-6.7.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b4938326284c4f1224178a560987b6cf8b4d38458b113d9b8c1db1a836e640a2", size = 240237 }, + { url = "https://files.pythonhosted.org/packages/5f/60/c3a5187bf66f6fb546ff4ab8fb5a077cbdd832d7b1908d4365c7f74a1917/multidict-6.7.1-cp314-cp314t-win32.whl", hash = "sha256:98655c737850c064a65e006a3df7c997cd3b220be4ec8fe26215760b9697d4d7", size = 48008 }, + { url = "https://files.pythonhosted.org/packages/0c/f7/addf1087b860ac60e6f382240f64fb99f8bfb532bb06f7c542b83c29ca61/multidict-6.7.1-cp314-cp314t-win_amd64.whl", hash = "sha256:497bde6223c212ba11d462853cfa4f0ae6ef97465033e7dc9940cdb3ab5b48e5", size = 53542 }, + { url = "https://files.pythonhosted.org/packages/4c/81/4629d0aa32302ef7b2ec65c75a728cc5ff4fa410c50096174c1632e70b3e/multidict-6.7.1-cp314-cp314t-win_arm64.whl", hash = "sha256:2bbd113e0d4af5db41d5ebfe9ccaff89de2120578164f86a5d17d5a576d1e5b2", size = 44719 }, + { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319 }, +] + +[[package]] +name = "nest-asyncio" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/83/f8/51569ac65d696c8ecbee95938f89d4abf00f47d58d48f6fbabfe8f0baefe/nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe", size = 7418 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/c4/c2971a3ba4c6103a3d10c4b0f24f461ddc027f0f09763220cf35ca1401b3/nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c", size = 5195 }, +] + +[[package]] +name = "openai" +version = "2.38.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "distro" }, + { name = "httpx" }, + { name = "jiter" }, + { name = "pydantic" }, + { name = "sniffio" }, + { name = "tqdm" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/12/cfa322c5f5dd8fa21aab9a7a8e979e7a11123800f86ca8d82eb68a83d213/openai-2.38.0.tar.gz", hash = "sha256:798694c6cf74145541fda94325b6f8f72d8e1fd0262cc137c8d728177a6a4ce3", size = 772764 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/bf/ccff9be562e24207716d04ef9dc931c76aff0c89a7265da43e2104d7fe06/openai-2.38.0-py3-none-any.whl", hash = "sha256:ec6661c57b2dcc47414a767e6e3335c7ed3d19c9696999283a3c82e95c756a3c", size = 1344910 }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b4/1c/125e1c936c0873796771b7f04f6c93b9f1bf5d424cea90fda94a99f61da8/opentelemetry_api-1.42.1.tar.gz", hash = "sha256:56c63bea9f77b62856be8c47600474acad853b2924b99b1687c4cb6297166716", size = 72296 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/ca/9520cc1f3dfbbd03ac5903bbf55833e257bc64b1cf30fa8b0d6df374d821/opentelemetry_api-1.42.1-py3-none-any.whl", hash = "sha256:51a69edacadbc03a8950ace1c4c21099cacc538820ac2c9e36277e78cebba714", size = 61311 }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.42.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/f7/b390bd9bfd703bf98a68fea1f27786c6872331fd617164a54b8a59bdc008/opentelemetry_sdk-1.42.1.tar.gz", hash = "sha256:8c834e8f8c9ba4171d4ec843d0cb8a67e4c7394d3f9e9297e582cbd9456ddbf7", size = 239262 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/6b/4287766cfbde577ae2272e8884abac325aeaac0d64f41c61d5b8cc595105/opentelemetry_sdk-1.42.1-py3-none-any.whl", hash = "sha256:083cd4bbfaa5aa7b5a9e552430d9951219967cfb27aa61feb13a77aba1fc839d", size = 170907 }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.63b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/93/99/4d7dd6df64795951413ce6e815f8cf1eb191daf7196ae86574589643d5f3/opentelemetry_semantic_conventions-0.63b1.tar.gz", hash = "sha256:3daf963611334b365e98a57438183eb012d3bfb40b2d931a9af613476b8701a9", size = 148340 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/7a/7fe66f5f3682b1dd47d88cc4e11f1c6c0966b737de2d16671146e23c39a5/opentelemetry_semantic_conventions-0.63b1-py3-none-any.whl", hash = "sha256:dfe5ef4dee82586b746f522b818ceb298d00b3d59f660042bd79404bff8d0682", size = 203713 }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195 }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 }, +] + +[[package]] +name = "portalocker" +version = "3.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/77/65b857a69ed876e1951e88aaba60f5ce6120c33703f7cb61a3c894b8c1b6/portalocker-3.2.0.tar.gz", hash = "sha256:1f3002956a54a8c3730586c5c77bf18fae4149e07eaf1c29fc3faf4d5a3f89ac", size = 95644 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4b/a6/38c8e2f318bf67d338f4d629e93b0b4b9af331f455f0390ea8ce4a099b26/portalocker-3.2.0-py3-none-any.whl", hash = "sha256:3cdc5f565312224bc570c49337bd21428bba0ef363bbcf58b9ef4a9f11779968", size = 22424 }, +] + +[[package]] +name = "posthog" +version = "7.15.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "backoff" }, + { name = "distro" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c5/ad/0eedae8cc9d2878d5b52c8607bd21f76101cfe4d875e5ff77fec9da3a83c/posthog-7.15.3.tar.gz", hash = "sha256:809dcaf08ca2d8bc0ea8228c28419181b74a79dfd1c0687a3d459a7bbe2e2953", size = 217645 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ad/b4/8dc673bed0f296c1acbb1107aef1c56db576731e894fe765206be5a91774/posthog-7.15.3-py3-none-any.whl", hash = "sha256:fd59fe4f5be637e4a2706b1457301d8308853ff23659036ecfcf6ac0a2d45eee", size = 254591 }, +] + +[[package]] +name = "propcache" +version = "0.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/44/c87281c333769159c50594f22610f77398a47ccbfbbf23074e744e86f87c/propcache-0.5.2.tar.gz", hash = "sha256:01c4fc7480cd0598bb4b57022df55b9ca296da7fc5a8760bd8451a7e63a7d427", size = 50208 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/f1/8a8cc1c2c7e7934ab77e0163414f736fadbc0f5e8dd9673b952355ac175b/propcache-0.5.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:74b70780220e2dd89175ca24b81b68b67c83db499ae611e7f2313cb329801c78", size = 90744 }, + { url = "https://files.pythonhosted.org/packages/c2/f4/651b1225e976bd1a2ba5cfba0c29d096581c2636b437e3a9a7ab6276270a/propcache-0.5.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a4840ab0ae0216d952f4b53dc6d0b992bfc2bedbfe360bdd9b548bc184c08959", size = 52033 }, + { url = "https://files.pythonhosted.org/packages/15/a8/8ede85d6aa1f79fc7dc2f8fd2c8d65920b8272c3892903c8a1affde48cfb/propcache-0.5.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c6844ba6364fb12f403928a82cfd295ab103a2b315c77c747b2dbe4a41894ea7", size = 52754 }, + { url = "https://files.pythonhosted.org/packages/7d/fe/b3551b41bbc2f5b5bb088fc6920567cd43101253e68fbaa261339eb96fe1/propcache-0.5.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2293949b855ce597f2826452d17c2d545fb5622379c4ea6fdf525e9b8e8a2511", size = 57573 }, + { url = "https://files.pythonhosted.org/packages/83/27/ab851ebd1b7172e3e161f5f8d39e315d54a91bea246f01f4d872d3376aef/propcache-0.5.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0fd59b5af35f74da48d905dcbad55449ba13be91823cb05a9bd590bbf5b61660", size = 60645 }, + { url = "https://files.pythonhosted.org/packages/95/7d/466b3d18022e9897cbda9c735c493c5bd747d7a4c6f5ea1480b4cec434b6/propcache-0.5.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:29f9309a2e42b0d273be006fdb4be2d6c39a47f6f57d8fb1cf9f81481df81b66", size = 61563 }, + { url = "https://files.pythonhosted.org/packages/27/1b/16ab7f2cf2041da2f60d156ba64c2484eadf9168075b4ff43c3ef60045af/propcache-0.5.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5aaa2b923c1944ac8febd6609cb373540a5563e7cbcb0fd770f75dace2eb817b", size = 58888 }, + { url = "https://files.pythonhosted.org/packages/0a/67/bb777ffd907633563bf35fd859c4ce97b0512c32f4633cf5d1eb7c33512b/propcache-0.5.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:66ea454f095ddf5b6b14f56c064c0941c4788be11e18d2464cf643bf7203ff67", size = 59253 }, + { url = "https://files.pythonhosted.org/packages/b9/42/64f8d90b73fd9cdc1499b48057ff6d9cd2a98a25734c9bb62ecf07e87061/propcache-0.5.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:95f1e3f4760d404b13c9976c0229b2b49a3c8e2c62a9ce92efdd2b11ada75e3f", size = 57558 }, + { url = "https://files.pythonhosted.org/packages/eb/02/dba5bc03c9041f2092ea55a449caf5dfe68352c6654511b29ba0654ddb69/propcache-0.5.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:85341b12b9d55bad0bded24cac341bb34289469e03a11f3f583ea1cc1db0326c", size = 55007 }, + { url = "https://files.pythonhosted.org/packages/14/c0/43f649c7aa2a77a3b100d84e9dea3a483120ecb608bfe36ce49eaff517fe/propcache-0.5.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:26a4dca084132874e639895c3135dfad5eb20bae209f62d1aeb31b03e601c3c0", size = 60355 }, + { url = "https://files.pythonhosted.org/packages/83/c0/435dafd27f1cb4a495381dae60e25883ccfe4020bb72818e8184c1678092/propcache-0.5.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3b199b9b2b3d6a7edf3183ba8a9a137a22b97f7df525feb5ae1eccf026d2a9c6", size = 59057 }, + { url = "https://files.pythonhosted.org/packages/53/ae/6e292df9135d659944e96cb3389258e4a663e5b2b5f6c217ef0ddc8d2f73/propcache-0.5.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:e59bc9e66329185b93dab73f210f1a37f81cb40f321501db8017c9aea15dba27", size = 61938 }, + { url = "https://files.pythonhosted.org/packages/0b/42/314ebc50d8159055411fd6b0bda322ff510e4b1f7d2e4927940ad0f6af20/propcache-0.5.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:552ffadf6ad409844bc5919c42a0a83d88314cedddaea0e41e80a8b8fffe881f", size = 59731 }, + { url = "https://files.pythonhosted.org/packages/b8/9b/2da6dee38871c3c8772fabc2758325a5c9077d6d18c597737dc04dd884cd/propcache-0.5.2-cp311-cp311-win32.whl", hash = "sha256:cd416c1de191973c52ff1a12a57446bfc7642797b282d7caf2162d7d1b8aa9a0", size = 38966 }, + { url = "https://files.pythonhosted.org/packages/42/4e/f17363fb58c0afe05b067361cb6d86ed2d29de6506779a27547c4d183075/propcache-0.5.2-cp311-cp311-win_amd64.whl", hash = "sha256:44e488ef40dbb452700b2b1f8188934121f6648f52c295055662d2191959ff82", size = 42135 }, + { url = "https://files.pythonhosted.org/packages/c6/eb/6af6685077d22e8b33358d3c548e3282706a0b3cd85044ffba4e5dd08e3b/propcache-0.5.2-cp311-cp311-win_arm64.whl", hash = "sha256:54adaa85a22078d1e306304a40984dc5be99d599bf3dc0a24dc98f7daeab89ab", size = 38381 }, + { url = "https://files.pythonhosted.org/packages/4a/cb/e27bc2b2737a0bb49962b275efa051e8f1c35a936df7d5139b6b658b7dc9/propcache-0.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806719138ecd720339a12410fb9614ac9b2b2d3a5fdf8235d56981c36f4039ba", size = 95887 }, + { url = "https://files.pythonhosted.org/packages/e6/13/b8ae04c59392f8d11c6cd9fb4011d1dc7c86b81225c770280300e259ffe1/propcache-0.5.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:db2b80ea58eab4f86b2beec3cc8b39e8ff9276ac20e96b7cce43c8ae84cd6b5a", size = 54654 }, + { url = "https://files.pythonhosted.org/packages/2c/7d/49777a3e20b55863d4794384a38acd460c04157b0a00f8602b0d508b8431/propcache-0.5.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e5cbfac9f61484f7e9f3597775500cd3ebe8274e9b050c38f9525c77c97520bf", size = 55190 }, + { url = "https://files.pythonhosted.org/packages/44/c7/085d0cd63062e84044e3f05797749c3f8e3938ff3aeb0eb2f69d43fafc91/propcache-0.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5dbc581d2814337da56222fab8dc5f161cd798a434e49bac27930aaef798e144", size = 59995 }, + { url = "https://files.pythonhosted.org/packages/9c/42/32cf8e3009e92b2645cf1e944f701e8ea4e924dffde1ee26db860bcbf7e4/propcache-0.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:857187f381f88c8e2fa2fe56ab94879d011b883d5a2ee5a1b60a8cd2a06846d9", size = 63422 }, + { url = "https://files.pythonhosted.org/packages/9e/1b/f112433f99fc979431b87a39ef169e3f8df070d99a72792c56d6937ac48b/propcache-0.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:178b4a2cdaac1818e2bf1c5a99b94383fa73ea5382e032a48dec07dc5668dc42", size = 64342 }, + { url = "https://files.pythonhosted.org/packages/14/15/5574111ae50dd6e879456888c0eadd4c5a869959775854e18e18a6b345f3/propcache-0.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6f328175a2cde1f0ff2c4ed8ce968b9dcfb55f3a7153f39e2957ed994da13476", size = 61639 }, + { url = "https://files.pythonhosted.org/packages/cc/da/4d775080b1490c0ae604acda868bd71aabe3a89ed16f2aa4339eb8a283e7/propcache-0.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5671d09a36b06d0fd4a3da0fccbcae360e9b1570924171a15e9e0997f0249fba", size = 61588 }, + { url = "https://files.pythonhosted.org/packages/04/ac/f076982cbe2195ee9cf32de5a1e46951d9fb399fc207f390562dd0fd8fb2/propcache-0.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:80168e2ebe4d3ec6599d10ad8f520304ae1cad9b6c5a95372aef1b66b7bfb53a", size = 60029 }, + { url = "https://files.pythonhosted.org/packages/70/60/189be62e0dd898dce3b331e1b8c7a543cd3a405ac0c81fe8ee8a9d5d77e1/propcache-0.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:45f11346f884bc47444f6e6647131055844134c3175b629f84952e2b5cd62b64", size = 56774 }, + { url = "https://files.pythonhosted.org/packages/ea/9e/93377b9c7939c1ffae98f878dee955efadfd638078bc86dbc21f9d52f651/propcache-0.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:8e778ebd44ef4f66ed60a0416b06b489687db264a9c0b3620362f26489492913", size = 63532 }, + { url = "https://files.pythonhosted.org/packages/14/f9/590ef6cfb9b8028d516d287812ece32bb0bc5f11fbb9c8bf6b2e6313fec8/propcache-0.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:c0cb9ed24c8964e172768d455a38254c2dd8a552905729ce006cad3d3dda59b1", size = 61592 }, + { url = "https://files.pythonhosted.org/packages/b4/5e/70958b3034c297a630bba2f17ca7abc2d5f39a803ad7e370ab79d1ecd022/propcache-0.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:1d1ad32d9d4355e2be65574fd0bfd3677e7066b009cd5b9b2dee8aa6a6393b33", size = 64788 }, + { url = "https://files.pythonhosted.org/packages/12/fd/77fe5936d8c3086ca9048f7f415f122ed82e53884a9ec193646b42deef06/propcache-0.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c80f4ba3e8f00189165999a742ee526ebeccedf6c3f7beb0c7df821e9772435a", size = 62514 }, + { url = "https://files.pythonhosted.org/packages/cf/74/66bd798b5b3be70aa1b391f5cc9d6a0a5532d7fd3b19ec0b213e72e6ad9d/propcache-0.5.2-cp312-cp312-win32.whl", hash = "sha256:8c7972d8f193740d9175f0998ab38717e6cd322d5935c5b0fef8c0d323fd9031", size = 39018 }, + { url = "https://files.pythonhosted.org/packages/61/7c/5c0d34aa3024694d6dcb9271cdbdd08c4e47c1c0ad95ec7e7bc74cdea145/propcache-0.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:d9ee8826a7d47863a08ac44e1a5f611a462eefc3a194b492da242128bec75b42", size = 42322 }, + { url = "https://files.pythonhosted.org/packages/4d/91/875812f1a3feb20ceba818ef39fbe4d92f1081e04ac815c822496d0d038b/propcache-0.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:2800a4a8ead6b28cccd1ec54b59346f0def7922ee1c7598e8499c733cfbb7c84", size = 38172 }, + { url = "https://files.pythonhosted.org/packages/c5/09/f049e45385503fe67db75a6b6186a7b9f0c3930366dc960522c312a825b1/propcache-0.5.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:099aaf4b4d1a02265b92a977edf00b5c4f63b3b17ac6de39b0d637c9cac0188a", size = 94457 }, + { url = "https://files.pythonhosted.org/packages/6b/65/83d1d05655baf63113731bd5a1008435e14f8d1e5a06cbe4ec5b23ad7a31/propcache-0.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:68ce1c44c7a813a7f71ea04315a8c7b330b63db99d059a797a4651bb6f69f117", size = 53835 }, + { url = "https://files.pythonhosted.org/packages/a9/12/a6ba6482bb5ea3260c000c9b20881c95fa11c6b30173715668259f844ed7/propcache-0.5.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fc299c129490f55f254cd90be0deca4764e36e9a7c08b4aa588479a3bbed3098", size = 54545 }, + { url = "https://files.pythonhosted.org/packages/a9/19/7fa086f5764c59ec8a8e157cd93aa8497acc00aba9dcdec56bfffb32602d/propcache-0.5.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6ae2198be502c10f09b2516e7b5d019816924bc3183a43ce792a7bd6625e6f4", size = 59886 }, + { url = "https://files.pythonhosted.org/packages/a1/e4/5d7663dc8235956c8f5281698a3af1d351d8820341ddd890f59d9a9127f2/propcache-0.5.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6041d31504dc1779d700e1edcfb08eea334b357620b06681a4eabb57a74e574e", size = 63261 }, + { url = "https://files.pythonhosted.org/packages/4a/4a/15a03adee24d6350da4292caeac44c34c033d2afe5e87eb370f38854560f/propcache-0.5.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7eabc04151c78a9f4d5bbb5f1faf571e4defeb4b585e0fe95b60ff2dbe4d3d7", size = 64184 }, + { url = "https://files.pythonhosted.org/packages/8b/c6/979176efdaa3d239e36d503d5af63a0a773b36662ed8f52e5b6a6d9fd40e/propcache-0.5.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4db0ba63d693afd40d249bd93f842b5f144f8fcbb83de05660373bcf30517b1d", size = 61534 }, + { url = "https://files.pythonhosted.org/packages/c8/22/63e8cd1bae4c2d2be6493b6b7d10566ddafad88137cfbc99964a1119853c/propcache-0.5.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dbcf7675229b35d31abb6547d8ebc8c27a830ac3f9a794edff6254873ec7c0a", size = 61500 }, + { url = "https://files.pythonhosted.org/packages/60/5a/28e5d9acbac1cc9ccb67045e8c1b943aa8d79fdf39c93bd73cacd68008ea/propcache-0.5.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d310c013aad2c72f1c3f2f8dd3279d460a858c551f97aeb8c63e4693cca7b4d2", size = 59994 }, + { url = "https://files.pythonhosted.org/packages/f3/40/db650677f554a95b9c01a7c9d93d629e93a15562f5deb4573c9ee136fed2/propcache-0.5.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:06187263ddad280d05b4d8a8b3bb7d164cbebd469236544a42e6d9b28ac6a4fa", size = 56884 }, + { url = "https://files.pythonhosted.org/packages/80/45/70b39b89516ff8b96bf732fa6fded8cef20f293cb1508690101c3c07ec51/propcache-0.5.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3115559b8effafd63b142ea5ed53d63a16ea6469cbc63dce4ee194b42db5d853", size = 63464 }, + { url = "https://files.pythonhosted.org/packages/f9/e2/fa59d3a89eac5534293124af4f1d0d0ada091ce4a0ab4610ce03fd2bdd8d/propcache-0.5.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c60462af8e6dc30c35407c7237ea908d777b22862bbee27bc4699c0d8bcdc45a", size = 61588 }, + { url = "https://files.pythonhosted.org/packages/0b/97/efb547a55c4bc7381cfb202d6a2239ac621045277bc1ea5dfd3a7f0516c0/propcache-0.5.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:40314bca9ac559716fe374094fc81c11dcc34b64fd6c585360f5775690505704", size = 64667 }, + { url = "https://files.pythonhosted.org/packages/92/56/f5c7d9b4b7595d5127da38974d791b2153f3d1eae6c674af3583ace92ad3/propcache-0.5.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:cfa21e036ce1e1db2be04ba3b85d2df1bb1702fa01932d984c5464c665228ff4", size = 62463 }, + { url = "https://files.pythonhosted.org/packages/bd/3b/484a3a65fc9f9f60c41dcd17b428bace5389544e2c680994534a20755066/propcache-0.5.2-cp313-cp313-win32.whl", hash = "sha256:f156a3529f38063b6dbaf356e15602a7f95f8055b1295a438433a6386f10463d", size = 38621 }, + { url = "https://files.pythonhosted.org/packages/1c/fd/3f0f10dba4dabad3bf53102be007abf55481067952bde0fdddff439e7c61/propcache-0.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:dfed59d0a5aeb01e242e66ff0300bc4a265a7c05f612d30016f0b60b1017d757", size = 41649 }, + { url = "https://files.pythonhosted.org/packages/90/ec/6ce619cc32bb500a482f811f9cd509368b4e58e638d13f2c68f370d6b475/propcache-0.5.2-cp313-cp313-win_arm64.whl", hash = "sha256:ba338430e87ceb9c8f0cf754de38a9860560261e56c00376debd628698a7364f", size = 37636 }, + { url = "https://files.pythonhosted.org/packages/1b/82/c1d268bbbf2ef981c5bf0fbbe746db617c66e3bcefe431a1aa8943fbe23a/propcache-0.5.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:a592f5f3da71c8691c788c13cb6734b6d17663d2e1cb8caddf0673d01ef8847d", size = 98872 }, + { url = "https://files.pythonhosted.org/packages/f4/d4/52c871e73e864e6b34c0e2d58ac1ec5ccd149497ddc7ad2137ae98323a35/propcache-0.5.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:6a997d0489e9668a384fcfd5061b857aa5361de73191cac204d04b889cfbbafa", size = 56257 }, + { url = "https://files.pythonhosted.org/packages/67/f0/9b90ca2a210b3d09bcfcd96ecd0f55545c091535abce2a45de2775cfd357/propcache-0.5.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:10734b5484ea113152ee25a91dccedf81631791805d2c9ccb054958e51842c94", size = 56696 }, + { url = "https://files.pythonhosted.org/packages/9d/0e/6e9d4ba07c8e56e21ddec1e75f12148142b21ca83a51871babce095334f4/propcache-0.5.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cafca7e56c12bb02ae16d283742bef25a61122e9dab2b5b3f2ccbe589ce32164", size = 62378 }, + { url = "https://files.pythonhosted.org/packages/65/19/c10badaa463dde8a27ce884f8ee2ec37e6035b7c9f5ff0c8f74f06f08dac/propcache-0.5.2-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f064f8d2b59177878b7615df1735cd8fe3462ed6be8c7b217d17a276489c2b7f", size = 65283 }, + { url = "https://files.pythonhosted.org/packages/b0/b6/93bea99ca80e19cef6512a8580e5b7857bbe09422d9daa7fd4ef5723306c/propcache-0.5.2-cp313-cp313t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f78abfa8dfc32376fd1aacf597b2f2fbbe0ea751419aee718af5d4f82537ef8c", size = 66616 }, + { url = "https://files.pythonhosted.org/packages/83/e4/5c7462e50625f051f37fb38b8224f7639f667184bbd34424ec83819bb1b7/propcache-0.5.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7467da8a9822bf1a55336f877340c5bcbd3c482afc43a99771169f74a26dedc", size = 63773 }, + { url = "https://files.pythonhosted.org/packages/ca/b6/99238894047b13c823be25027e736626cd414a52a5e30d2c3347c2733529/propcache-0.5.2-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a6ddc6ac9e25de626c1f129c1b467d7ecd33ce2237d3fd0c4e429feef0a7ee1f", size = 63664 }, + { url = "https://files.pythonhosted.org/packages/85/1e/a3a1a63116a2b8edb415a8bb9a6f0c34bd03830b1e18e8ce2904e1dc1cf4/propcache-0.5.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2f22cbbac9e26a8e864c0985ff1268d5d939d53d9d9411a9824279097e03a2cb", size = 62643 }, + { url = "https://files.pythonhosted.org/packages/e4/03/893cf147de2fc6543c5eaa07ad833170e7e2a2385725bbebe8c0503723bb/propcache-0.5.2-cp313-cp313t-musllinux_1_2_armv7l.whl", hash = "sha256:fc76378c62a0f04d0cd82fbb1a2cd2d7e28fcb40d5873f28a6c44e388aaa2751", size = 59595 }, + { url = "https://files.pythonhosted.org/packages/86/3b/04c1a2e12c57766568ba75ba72b3bf2042818d4c1425fab6fc07155c7cff/propcache-0.5.2-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:acd2c8edba48e31e58a363b8cf4e5c7db3b04b3f9e371f601df30d9b0d244836", size = 65711 }, + { url = "https://files.pythonhosted.org/packages/1c/34/80f8d0099f8d6bacc4de1624c85672681c8cd1149ca2da0e38fd120b817f/propcache-0.5.2-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:452b5065457eb9991ec5eb38ff41d6cd4c991c9ac7c531c4d5849ae473a9a13f", size = 64247 }, + { url = "https://files.pythonhosted.org/packages/f3/1a/8b08f3a5f1037e9e370c55883ceeeee0f6dd0416fb2d2d67b8bfc91f2a79/propcache-0.5.2-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:3430bb2bfe1331885c427745a751e774ee679fd4344f80b97bf879815fe8fa55", size = 67102 }, + { url = "https://files.pythonhosted.org/packages/34/68/8bdb7bb7756d76e005490649d10e4a8369e610c74d619f71e1aedf889e9c/propcache-0.5.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cef6cea3922890dd6c9654971001fa797b526c16ab5e1e46c05fd6f877be7568", size = 64964 }, + { url = "https://files.pythonhosted.org/packages/0a/aa/50fb0b5d3968b61a510926ff8b8465f1d6e976b3ab74496d7a4b9fc42515/propcache-0.5.2-cp313-cp313t-win32.whl", hash = "sha256:72d61e16dd78228b58c5d47be830ff3da7e5f139abdf0aef9d86cde1c5cf2191", size = 42546 }, + { url = "https://files.pythonhosted.org/packages/ae/4c/0ddbae64321bd4a95bcbfc19307238016b5b1fee645c84626c8d539e5b74/propcache-0.5.2-cp313-cp313t-win_amd64.whl", hash = "sha256:0958834041a0166d343b8d2cedcd8bcbaeb4fdbe0cf08320c5379f143c3be6e7", size = 46330 }, + { url = "https://files.pythonhosted.org/packages/00/d9/9cddc8efb78d8af264c5ec9f6d10b62f57c515feda8d321595f56010fb23/propcache-0.5.2-cp313-cp313t-win_arm64.whl", hash = "sha256:6de8bd93ddde9b992cf2b2e0d796d501a19026b5b9fd87356d7d0779531a8d96", size = 40521 }, + { url = "https://files.pythonhosted.org/packages/e2/ea/23ee535d90ce8bcc465a3028eb3cc0ce3bd1005f4bb27710b30587de798d/propcache-0.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:46088abff4cba581dea21ae0467a480526cb25aa5f3c269e909f800328bc3999", size = 94662 }, + { url = "https://files.pythonhosted.org/packages/b5/06/c5a52f419b5d8972f8d46a7577476090d8e3263ff589ce40b5ca4968d5be/propcache-0.5.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fc88b26f08d634f7bc819a7852e5214f5802641ab8d9fd5326892292eee1993e", size = 53928 }, + { url = "https://files.pythonhosted.org/packages/63/b1/4260d67d6bd85e58a66b72d54ce15d5de789b6f3870cc6bedf8ff9667401/propcache-0.5.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97797ebb098e670a2f92dd66f32897e30d7615b14e7f59711de23e30a9072539", size = 54650 }, + { url = "https://files.pythonhosted.org/packages/70/06/2f46c318e3307cd7a6a7481def374ce838c0fe20084b39dd54b0879d0e99/propcache-0.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ba57fffe4ac99c5d30076161b5866336d97600769bad35cc68f7774b15298a4e", size = 59912 }, + { url = "https://files.pythonhosted.org/packages/4c/29/fe1aebec2ce57ab985a9c382bded1124431f85078113aa222c5d278430d4/propcache-0.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:583c19759d9eec1e5b69e2fbef36a7d9c326041be9746cb822d335c8cedc2979", size = 63300 }, + { url = "https://files.pythonhosted.org/packages/b4/18/2334b26768b6c82be8c69e83671b767d5ef426aa09b0cba6c2ea47816774/propcache-0.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d0326e2e5e1f3163fa306c834e48e8d490e5fae607a097a40c0648109b47ba80", size = 64208 }, + { url = "https://files.pythonhosted.org/packages/2b/76/7f1bfd6afff4c5e38e36a3c6d68eb5f4b7311ea80baf693db78d95b603c4/propcache-0.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e00820e192c8dbebcafb383ebbf99030895f09905e7a0eb2e0340a0bcc2bc825", size = 61633 }, + { url = "https://files.pythonhosted.org/packages/c4/46/b3ff8aba2b4953a3e50de2cf72f1b5748b8eca93b15f3dc2c84339084c09/propcache-0.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c66afea89b1e43725731d2004732a046fe6fe955d51f952c3e95a7314a284a39", size = 61724 }, + { url = "https://files.pythonhosted.org/packages/c5/01/814cfcafbcff954f94c01cf30e097ddc88a076b5440fbcf4570753437d40/propcache-0.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d4dc37dec6c6cdad0b57881a5658fd14fbf53e333b1a86cf86559f190e1d9ec4", size = 60069 }, + { url = "https://files.pythonhosted.org/packages/da/68/5c6f7622d510cc666a300687e06fd060c1a43361c0c9b20d284f06d8096a/propcache-0.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:5570dbcc97571c15f68068e529c92715a12f8d54030e272d264b377e22bd17a5", size = 57099 }, + { url = "https://files.pythonhosted.org/packages/55/27/9cb0b4c679124085327957d42521c99dba04c88c90c3e55a6f0b633ebccc/propcache-0.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:f814362777a9f841adddb200ecdf8f5cb1e5a3c4b7a86378edbd6ccb26edd702", size = 63391 }, + { url = "https://files.pythonhosted.org/packages/f0/9d/7258aaa5bdf60fc6f27591eef6fe52768cb0beda7140be477c8b12c9794a/propcache-0.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:196913dea116aeb5a2ba95af4ddcb7ea85559ae07d8eee8751688310d09168c3", size = 61626 }, + { url = "https://files.pythonhosted.org/packages/8e/0d/41c602003e8a9b16fe1e7eadf62c7bfba9d5474370b24200bf48b315f45f/propcache-0.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6e7b8719005dd1175be4ab1cd25e9b98659a5e0347331506ec6760d2773a7fb5", size = 64781 }, + { url = "https://files.pythonhosted.org/packages/8b/f3/38e66b1856e9bd079deea015bc4a55f7767c0e4db2f7dcf69e7e680ba4ce/propcache-0.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:51f96d685ab16e88cab128cd37a52c5da540809c8b879fa047731bfcb4ad35a4", size = 62570 }, + { url = "https://files.pythonhosted.org/packages/95/ca/bbfe9b910ce57dde8bb4876b4520fc02a4e89497c10de26be936758a3aaa/propcache-0.5.2-cp314-cp314-win32.whl", hash = "sha256:cc6fc3cc62e8501d3ed62894425040d2728ecddb1ed072737a5c70bd537aa9f0", size = 39436 }, + { url = "https://files.pythonhosted.org/packages/61/d2/45c9defbaa1ea297035d9d4cce9e8f80daafbf19319c6007f157c6256ea9/propcache-0.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:81e3a30b0bb60caa22033dd0f8a3618d1d67356212514f62c57db75cb0ef410c", size = 42373 }, + { url = "https://files.pythonhosted.org/packages/44/68/9ea5103f41d5217d7d6ec24db90018e23aebec070c3f9a6e54d12b841fd8/propcache-0.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:0d2c9bf8528f135dbb805ce027567e09164f7efa51a2be07458a2c0420f292d0", size = 38554 }, + { url = "https://files.pythonhosted.org/packages/8a/81/fadf555f42d3b762eea8a53950b0489fdc0aa9da5f8ed9e10ce0a4e01b48/propcache-0.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4bc8ff1feffc6a61c7002ffe84634c41b822e104990ae009f44a0834430070bb", size = 99395 }, + { url = "https://files.pythonhosted.org/packages/f5/c9/c61e134a686949cf7971af3a390148b1156f7be81c73bc0cd12c873e2d48/propcache-0.5.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79aa3ff0a9b566633b642fa9caf7e21ed1c13d6feca718187873f199e1514078", size = 56653 }, + { url = "https://files.pythonhosted.org/packages/cb/73/daf935ea7048ddd7ec8eec5345b4a40b619d2d178b3c0a0900796bc3c794/propcache-0.5.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1b31822f4474c4036bae62de9402710051d431a606d6a0f907fec79935a071aa", size = 56914 }, + { url = "https://files.pythonhosted.org/packages/79/9f/aba959b435ea18617edd7cf0a7ad0b9c574b8fc7e3d2cd55fb59cb255d33/propcache-0.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:13fef48778b5a2a756523fdb781326b028ca75e32858b04f2cdd19f394564917", size = 62567 }, + { url = "https://files.pythonhosted.org/packages/6c/a1/859942de9a791ff42f6141736f5b37749b8f53e65edfa49638c67dd67e6a/propcache-0.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8b73ab70f1a3351fbc71f663b3e645af6dd0329100c353081cf69c37433fc6fe", size = 65542 }, + { url = "https://files.pythonhosted.org/packages/b5/61/315bc0fd6c0fc7f80a528b8afd209e5fc4a875ea79571b91b8f50f442907/propcache-0.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5538d2c13d93e4698af7e092b57bc7298fd35d1d58e656ae18f23ee0d0378e03", size = 66845 }, + { url = "https://files.pythonhosted.org/packages/47/f7/9f8122e3132e8e354ac41975ef8f1099be7d5a16bc7ae562734e993665c0/propcache-0.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cd645f03898405cabe694fb8bc35241e3a9c332ec85627584fe3de201452b335", size = 63985 }, + { url = "https://files.pythonhosted.org/packages/c8/54/c317819ec157cbf6f35df9df9657a6f82daf34d5faf15948b2f639c2192e/propcache-0.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a473b3440261e0c60706e732b2ed2f517857344fc21bf48fdfe211e2d98eb285", size = 63999 }, + { url = "https://files.pythonhosted.org/packages/5a/56/387e3f7dfce0a9233df41fb888aa1c30222cb4bbbf09537c02dd9bd85fe2/propcache-0.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7afa37062e6650640e932e4cc9297d81f9f42d9944029cc386b8247dea4da837", size = 62779 }, + { url = "https://files.pythonhosted.org/packages/a1/9c/596784cb5824ed61ee960d3f8655a3f0993e107c6e98ab6c818b7fb92ccb/propcache-0.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:8a90efd5777e996e42d568db9ac740b944d691e565cbfd31b2f7832f9184b2b8", size = 59796 }, + { url = "https://files.pythonhosted.org/packages/c2/3d/1a6cfa1726a48542c1e8784a0761421476a5b68e09b7f36bf95eb954aaba/propcache-0.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:f19bb891234d72535764d703bfed1153cc34f4214d5bd7150aee1eec9e8f4366", size = 66023 }, + { url = "https://files.pythonhosted.org/packages/e4/0e/05fd6990369477076e4e280bcb970de760fddf0161a46e988bc95f7940ec/propcache-0.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:32775082acd2d807ee3db715c7770d38767b817870acfa08c29e057f3c4d5b56", size = 64448 }, + { url = "https://files.pythonhosted.org/packages/cd/86/5f8da315a4309c62c10c0b2516b17492d5d3bbe1bb862b96604db67e2a37/propcache-0.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:9282fb1a3bccd038da9f768b927b24a0c753e466c086b7c4f3c6982851eefb2d", size = 67329 }, + { url = "https://files.pythonhosted.org/packages/da/d3/3368efe79ab21f0cdf86ef49895811c9cc933131d4cde1f28a624e22e712/propcache-0.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cc49723e2f60d6b32a0f0b08a3fd6d13203c07f1cd9566cfce0f12a917c967a2", size = 65172 }, + { url = "https://files.pythonhosted.org/packages/d5/07/127e8b0bacfb325396196f9d976a22453049b89b9b2b08477cc3145faa44/propcache-0.5.2-cp314-cp314t-win32.whl", hash = "sha256:2d7aa89ebca5acc98cba9d1472d976e394782f587bad6661003602a619fd1821", size = 43813 }, + { url = "https://files.pythonhosted.org/packages/88/fb/46dad6c0ae49ed230ab1b16c890c2b6314e2403e6c412976f4a72d64a527/propcache-0.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:d447bb0b3054be5818458fbb171208b1d9ff11eba14e18ca18b90cbb45767370", size = 47764 }, + { url = "https://files.pythonhosted.org/packages/e7/c4/a47d0a63aa309d10d59ede6e9d4cff03a344a79d1f0f4cd0cd74997b53e0/propcache-0.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:fe67a3d11cd9b4efabfa45c3d00ffba2b26811442a73a581a94b67c2b5faccf6", size = 41140 }, + { url = "https://files.pythonhosted.org/packages/3a/ed/1cdcab6ba3d6ab7feca11fc14f0eeea80755bb53ef4e892079f31b10a25f/propcache-0.5.2-py3-none-any.whl", hash = "sha256:be1ddfcbb376e3de5d2e2db1d58d6d67463e6b4f9f040c000de8e300295465fe", size = 14036 }, +] + +[[package]] +name = "pycparser" +version = "3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/7d/92392ff7815c21062bea51aa7b87d45576f649f16458d78b7cf94b9ab2e6/pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29", size = 103492 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172 }, +] + +[[package]] +name = "pycryptodome" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152 }, + { url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348 }, + { url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033 }, + { url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142 }, + { url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384 }, + { url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237 }, + { url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898 }, + { url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197 }, + { url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600 }, + { url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740 }, + { url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685 }, + { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627 }, + { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362 }, + { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625 }, + { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954 }, + { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534 }, + { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853 }, + { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465 }, + { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414 }, + { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484 }, + { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636 }, + { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675 }, +] + +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262 }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/fa/6d7708d2cfc1a832acb6aeb0cd16e801902df8a0f583bb3b4b527fde022e/pydantic_core-2.46.4-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594", size = 2111872 }, + { url = "https://files.pythonhosted.org/packages/ae/6f/aa064a3e74b5745afbdf250594f38e7ead05e2d651bcb35994b9417a0d4d/pydantic_core-2.46.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c", size = 1948255 }, + { url = "https://files.pythonhosted.org/packages/43/3a/41114a9f7569b84b4d84e7a018c57c56347dac30c0d4a872946ec4e36c46/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826", size = 1972827 }, + { url = "https://files.pythonhosted.org/packages/ef/25/1ab42e8048fe551934d9884e8d64daa7e990ad386f310a15981aeb6a5b08/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04", size = 2041051 }, + { url = "https://files.pythonhosted.org/packages/94/c2/1a934597ddf08da410385b3b7aae91956a5a76c635effef456074fad7e88/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e", size = 2221314 }, + { url = "https://files.pythonhosted.org/packages/02/6d/9e8ad178c9c4df27ad3c8f25d1fe2a7ab0d2ba0559fad4aee5d3d1f16771/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3", size = 2285146 }, + { url = "https://files.pythonhosted.org/packages/80/50/540cd3aeefc041beb111125c4bff779831a2111fc6b15a9138cda277d32c/pydantic_core-2.46.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4", size = 2089685 }, + { url = "https://files.pythonhosted.org/packages/6b/a4/b440ad35f05f6a38f89fa0f149accb3f0e02be94ca5e15f3c449a61b4bc9/pydantic_core-2.46.4-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398", size = 2115420 }, + { url = "https://files.pythonhosted.org/packages/99/61/de4f55db8dfd57bfdfa9a12ec90fe1b57c4f41062f7ca86f08586b3e0ac0/pydantic_core-2.46.4-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3", size = 2165122 }, + { url = "https://files.pythonhosted.org/packages/f7/52/7c529d7bdb2d1068bd52f51fe32572c8301f9a4febf1948f10639f1436f5/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848", size = 2182573 }, + { url = "https://files.pythonhosted.org/packages/37/b3/7c40325848ba78247f2812dcf9c7274e38cd801820ca6dd9fe63bcfb0eb4/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3", size = 2317139 }, + { url = "https://files.pythonhosted.org/packages/d9/37/f913f81a657c865b75da6c0dbed79876073c2a43b5bd9edbe8da785e4d49/pydantic_core-2.46.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109", size = 2360433 }, + { url = "https://files.pythonhosted.org/packages/c4/67/6acaa1be2567f9256b056d8477158cac7240813956ce86e49deae8e173b4/pydantic_core-2.46.4-cp311-cp311-win32.whl", hash = "sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda", size = 1985513 }, + { url = "https://files.pythonhosted.org/packages/aa/e6/c505f83dfeda9a2e5c995cfd872949e4d05e12f7feb3dca72f633daefa94/pydantic_core-2.46.4-cp311-cp311-win_amd64.whl", hash = "sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33", size = 2071114 }, + { url = "https://files.pythonhosted.org/packages/0f/da/7a263a96d965d9d0df5e8de8a475f33495451117035b09acb110288c381f/pydantic_core-2.46.4-cp311-cp311-win_arm64.whl", hash = "sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d", size = 2044298 }, + { url = "https://files.pythonhosted.org/packages/ce/8c/af022f0af448d7747c5154288d46b5f2bc5f17366eaa0e23e9aa04d59f3b/pydantic_core-2.46.4-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2", size = 2106158 }, + { url = "https://files.pythonhosted.org/packages/19/95/6195171e385007300f0f5574592e467c568becce2d937a0b6804f218bc49/pydantic_core-2.46.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f", size = 1951724 }, + { url = "https://files.pythonhosted.org/packages/8e/bc/f47d1ff9cbb1620e1b5b697eef06010035735f07820180e74178226b27b3/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7", size = 1975742 }, + { url = "https://files.pythonhosted.org/packages/5b/11/9b9a5b0306345664a2da6410877af6e8082481b5884b3ddd78d47c6013ce/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7", size = 2052418 }, + { url = "https://files.pythonhosted.org/packages/f1/b7/a65fec226f5d78fc39f4a13c4cc0c768c22b113438f60c14adc9d2865038/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712", size = 2232274 }, + { url = "https://files.pythonhosted.org/packages/68/f0/92039db98b907ef49269a8271f67db9cb78ae2fc68062ef7e4e77adb5f61/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4", size = 2309940 }, + { url = "https://files.pythonhosted.org/packages/5f/97/2aab507d3d00ca626e8e57c1eac6a79e4e5fbcc63eb99733ff55d1717f65/pydantic_core-2.46.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce", size = 2094516 }, + { url = "https://files.pythonhosted.org/packages/22/37/a8aca44d40d737dde2bc05b3c6c07dff0de07ce6f82e9f3167aeaf4d5dea/pydantic_core-2.46.4-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987", size = 2136854 }, + { url = "https://files.pythonhosted.org/packages/24/99/fcef1b79238c06a8cbec70819ac722ba76e02bc8ada9b0fd66eba40da01b/pydantic_core-2.46.4-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b", size = 2180306 }, + { url = "https://files.pythonhosted.org/packages/ae/6c/fc44000918855b42779d007ae63b0532794739027b2f417321cddbc44f6a/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458", size = 2190044 }, + { url = "https://files.pythonhosted.org/packages/6b/65/d9cadc9f1920d7a127ad2edba16c1db7916e59719285cd6c94600b0080ba/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b", size = 2329133 }, + { url = "https://files.pythonhosted.org/packages/d0/cf/c873d91679f3a30bcf5e7ac280ce5573483e72295307685120d0d5ad3416/pydantic_core-2.46.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c", size = 2374464 }, + { url = "https://files.pythonhosted.org/packages/47/bd/6f2fc8188f31bf10590f1e98e7b306336161fac930a8c514cd7bd828c7dc/pydantic_core-2.46.4-cp312-cp312-win32.whl", hash = "sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894", size = 1974823 }, + { url = "https://files.pythonhosted.org/packages/40/8c/985c1d41ea1107c2534abd9870e4ed5c8e7669b5c308297835c001e7a1c4/pydantic_core-2.46.4-cp312-cp312-win_amd64.whl", hash = "sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89", size = 2072919 }, + { url = "https://files.pythonhosted.org/packages/c4/ba/f463d006e0c47373ca7ec5e1a261c59dc01ef4d62b2657af925fb0deee3a/pydantic_core-2.46.4-cp312-cp312-win_arm64.whl", hash = "sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a", size = 2027604 }, + { url = "https://files.pythonhosted.org/packages/51/a2/5d30b469c5267a17b39dec53208222f76a8d351dfac4af661888c5aee77d/pydantic_core-2.46.4-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008", size = 2106306 }, + { url = "https://files.pythonhosted.org/packages/c1/81/4fa520eaffa8bd7d1525e644cd6d39e7d60b1592bc5b516693c7340b50f1/pydantic_core-2.46.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4", size = 1951906 }, + { url = "https://files.pythonhosted.org/packages/03/d5/fd02da45b659668b05923b17ba3a0100a0a3d5541e3bd8fcc4ecb711309e/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76", size = 1976802 }, + { url = "https://files.pythonhosted.org/packages/21/f2/95727e1368be3d3ed485eaab7adbd7dda408f33f7a36e8b48e0144002b91/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3", size = 2052446 }, + { url = "https://files.pythonhosted.org/packages/9c/86/5d99feea3f77c7234b8718075b23db11532773c1a0dbd9b9490215dc2eeb/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76", size = 2232757 }, + { url = "https://files.pythonhosted.org/packages/d2/3a/508ac615935ef7588cf6d9e9b91309fdc2da751af865e02a9098de88258c/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4", size = 2309275 }, + { url = "https://files.pythonhosted.org/packages/07/f8/41db9de19d7987d6b04715a02b3b40aea467000275d9d758ffaa31af7d50/pydantic_core-2.46.4-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a", size = 2094467 }, + { url = "https://files.pythonhosted.org/packages/2c/e2/f35033184cb11d0052daf4416e8e10a502ea2ac006fc4f459aee872727d1/pydantic_core-2.46.4-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262", size = 2134417 }, + { url = "https://files.pythonhosted.org/packages/7e/7b/6ceeb1cc90e193862f444ebe373d8fdf613f0a82572dde03fb10734c6c71/pydantic_core-2.46.4-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e", size = 2179782 }, + { url = "https://files.pythonhosted.org/packages/5a/f2/c8d7773ede6af08036423a00ae0ceffce266c3c52a096c435d68c896083f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd", size = 2188782 }, + { url = "https://files.pythonhosted.org/packages/59/31/0c864784e31f09f05cdd87606f08923b9c9e7f6e51dd27f20f62f975ce9f/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be", size = 2328334 }, + { url = "https://files.pythonhosted.org/packages/c2/eb/4f6c8a41efa30baa755590f4141abf3a8c370fab610915733e74134a7270/pydantic_core-2.46.4-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d", size = 2372986 }, + { url = "https://files.pythonhosted.org/packages/5b/24/b375a480d53113860c299764bfe9f349a3dc9108b3adc0d7f0d786492ebf/pydantic_core-2.46.4-cp313-cp313-win32.whl", hash = "sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb", size = 1973693 }, + { url = "https://files.pythonhosted.org/packages/7e/e8/cff247591966f2d22ec8c003cd7587e27b7ba7b81ab2fb888e3ab75dc285/pydantic_core-2.46.4-cp313-cp313-win_amd64.whl", hash = "sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292", size = 2071819 }, + { url = "https://files.pythonhosted.org/packages/c6/1a/f4aee670d5670e9e148e0c82c7db98d780be566c6e6a97ee8035528ca0b3/pydantic_core-2.46.4-cp313-cp313-win_arm64.whl", hash = "sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d", size = 2027411 }, + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079 }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179 }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926 }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785 }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733 }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534 }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732 }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627 }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141 }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325 }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990 }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978 }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354 }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238 }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251 }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593 }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226 }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605 }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777 }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641 }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404 }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219 }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594 }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542 }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146 }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309 }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736 }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575 }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624 }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325 }, + { url = "https://files.pythonhosted.org/packages/11/cb/428de0385b6c8d44b716feba566abfacfbd23ee3c4439faa789a1456242f/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0", size = 2112782 }, + { url = "https://files.pythonhosted.org/packages/0b/b5/6a17bdadd0fc1f170adfd05a20d37c832f52b117b4d9131da1f41bb097ce/pydantic_core-2.46.4-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7", size = 1952146 }, + { url = "https://files.pythonhosted.org/packages/2a/dc/03734d80e362cd43ef65428e9de77c730ce7f2f11c60d2b1e1b39f0fbf99/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2", size = 2134492 }, + { url = "https://files.pythonhosted.org/packages/de/df/5e5ffc085ed07cc22d298134d3d911c63e91f6a0eb91fe646750a3209910/pydantic_core-2.46.4-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9", size = 2156604 }, + { url = "https://files.pythonhosted.org/packages/81/44/6e112a4253e56f5705467cbab7ab5e91ee7398ba3d56d358635958893d3e/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf", size = 2183828 }, + { url = "https://files.pythonhosted.org/packages/ac/ad/5565071e937d8e752842ac241463944c9eb14c87e2d269f2658a5bd05e98/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30", size = 2310000 }, + { url = "https://files.pythonhosted.org/packages/4f/c3/66883a5cec183e7fba4d024b4cbbe61851a63750ef606b0afecc46d1f2bf/pydantic_core-2.46.4-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc", size = 2361286 }, + { url = "https://files.pythonhosted.org/packages/4b/2d/69abac8f838090bbecd5df894befb2c2619e7996a98ddb949db9f3b93225/pydantic_core-2.46.4-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983", size = 2193071 }, +] + +[[package]] +name = "pydantic-settings" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/60/1d1e59c9c90d54591469ada7d268251f71c24bdb765f1a8a832cee8c6653/pydantic_settings-2.14.1.tar.gz", hash = "sha256:e874d3bec7e787b0c9958277956ed9b4dd5de6a80e162188fdaff7c5e26fd5fa", size = 235551 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/8d/f1af3832f5e6eb13ba94ee809e72b8ecb5eef226d27ee0bef7d963d943c7/pydantic_settings-2.14.1-py3-none-any.whl", hash = "sha256:6e3c7edfd8277687cdc598f56e5cff0e9bfff0910a3749deaa8d4401c3a2b9de", size = 60964 }, +] + +[[package]] +name = "pyfiglet" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c8/e3/0a86276ad2c383ce08d76110a8eec2fe22e7051c4b8ba3fa163a0b08c428/pyfiglet-1.0.4.tar.gz", hash = "sha256:db9c9940ed1bf3048deff534ed52ff2dafbbc2cd7610b17bb5eca1df6d4278ef", size = 1560615 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/5c/fe9f95abd5eaedfa69f31e450f7e2768bef121dbdf25bcddee2cd3087a16/pyfiglet-1.0.4-py3-none-any.whl", hash = "sha256:65b57b7a8e1dff8a67dc8e940a117238661d5e14c3e49121032bd404d9b2b39f", size = 1806118 }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151 }, +] + +[[package]] +name = "pyjwt" +version = "2.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274 }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + +[[package]] +name = "pytest" +version = "9.0.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249 }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075 }, +] + +[[package]] +name = "pytest-repeat" +version = "0.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/d4/69e9dbb9b8266df0b157c72be32083403c412990af15c7c15f7a3fd1b142/pytest_repeat-0.9.4.tar.gz", hash = "sha256:d92ac14dfaa6ffcfe6917e5d16f0c9bc82380c135b03c2a5f412d2637f224485", size = 6488 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/d4/8b706b81b07b43081bd68a2c0359fe895b74bf664b20aca8005d2bb3be71/pytest_repeat-0.9.4-py3-none-any.whl", hash = "sha256:c1738b4e412a6f3b3b9e0b8b29fcd7a423e50f87381ad9307ef6f5a8601139f3", size = 4180 }, +] + +[[package]] +name = "pytest-rerunfailures" +version = "16.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/f0/74f8e685be7ecd1572c1256132f18fce3a665d7e07649a3f23b7eb2d3bec/pytest_rerunfailures-16.3.tar.gz", hash = "sha256:37c9b1231c8083e9f4e724f50f7a21241822f9516c15c700ebbf218d6452355c", size = 34148 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/98/58a71d68d3126d7f6a6ed1944c37ec207a4ff3dc66cad3bed7b59d38df61/pytest_rerunfailures-16.3-py3-none-any.whl", hash = "sha256:6bdfb8ffb46c46072e6c16bdedee38b6c13eac620d9415ed5b63152cbf283170", size = 15396 }, +] + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396 }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101 }, +] + +[[package]] +name = "python-multipart" +version = "0.0.29" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/fe/70bd71a6738b09a0bdf6480ca6436b167469ca4578b2a0efbe390b4b0e70/python_multipart-0.0.29.tar.gz", hash = "sha256:643e93849196645e2dbdd81a0f8829a23123ad7f797a84a364c6fb3563f18904", size = 45678 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/cb/769cfc37177252872a45a71f3fbdde9d51b471a3f3c14bfe95dde3407386/python_multipart-0.0.29-py3-none-any.whl", hash = "sha256:2ddcc971cef266225f54f552d8fa10bcfbb1f14446caec199060daac59ff2d69", size = 29640 }, +] + +[[package]] +name = "pywin32" +version = "311" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/af/449a6a91e5d6db51420875c54f6aff7c97a86a3b13a0b4f1a5c13b988de3/pywin32-311-cp311-cp311-win32.whl", hash = "sha256:184eb5e436dea364dcd3d2316d577d625c0351bf237c4e9a5fabbcfa5a58b151", size = 8697031 }, + { url = "https://files.pythonhosted.org/packages/51/8f/9bb81dd5bb77d22243d33c8397f09377056d5c687aa6d4042bea7fbf8364/pywin32-311-cp311-cp311-win_amd64.whl", hash = "sha256:3ce80b34b22b17ccbd937a6e78e7225d80c52f5ab9940fe0506a1a16f3dab503", size = 9508308 }, + { url = "https://files.pythonhosted.org/packages/44/7b/9c2ab54f74a138c491aba1b1cd0795ba61f144c711daea84a88b63dc0f6c/pywin32-311-cp311-cp311-win_arm64.whl", hash = "sha256:a733f1388e1a842abb67ffa8e7aad0e70ac519e09b0f6a784e65a136ec7cefd2", size = 8703930 }, + { url = "https://files.pythonhosted.org/packages/e7/ab/01ea1943d4eba0f850c3c61e78e8dd59757ff815ff3ccd0a84de5f541f42/pywin32-311-cp312-cp312-win32.whl", hash = "sha256:750ec6e621af2b948540032557b10a2d43b0cee2ae9758c54154d711cc852d31", size = 8706543 }, + { url = "https://files.pythonhosted.org/packages/d1/a8/a0e8d07d4d051ec7502cd58b291ec98dcc0c3fff027caad0470b72cfcc2f/pywin32-311-cp312-cp312-win_amd64.whl", hash = "sha256:b8c095edad5c211ff31c05223658e71bf7116daa0ecf3ad85f3201ea3190d067", size = 9495040 }, + { url = "https://files.pythonhosted.org/packages/ba/3a/2ae996277b4b50f17d61f0603efd8253cb2d79cc7ae159468007b586396d/pywin32-311-cp312-cp312-win_arm64.whl", hash = "sha256:e286f46a9a39c4a18b319c28f59b61de793654af2f395c102b4f819e584b5852", size = 8710102 }, + { url = "https://files.pythonhosted.org/packages/a5/be/3fd5de0979fcb3994bfee0d65ed8ca9506a8a1260651b86174f6a86f52b3/pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d", size = 8705700 }, + { url = "https://files.pythonhosted.org/packages/e3/28/e0a1909523c6890208295a29e05c2adb2126364e289826c0a8bc7297bd5c/pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d", size = 9494700 }, + { url = "https://files.pythonhosted.org/packages/04/bf/90339ac0f55726dce7d794e6d79a18a91265bdf3aa70b6b9ca52f35e022a/pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a", size = 8709318 }, + { url = "https://files.pythonhosted.org/packages/c9/31/097f2e132c4f16d99a22bfb777e0fd88bd8e1c634304e102f313af69ace5/pywin32-311-cp314-cp314-win32.whl", hash = "sha256:b7a2c10b93f8986666d0c803ee19b5990885872a7de910fc460f9b0c2fbf92ee", size = 8840714 }, + { url = "https://files.pythonhosted.org/packages/90/4b/07c77d8ba0e01349358082713400435347df8426208171ce297da32c313d/pywin32-311-cp314-cp314-win_amd64.whl", hash = "sha256:3aca44c046bd2ed8c90de9cb8427f581c479e594e99b5c0bb19b29c10fd6cb87", size = 9656800 }, + { url = "https://files.pythonhosted.org/packages/c0/d2/21af5c535501a7233e734b8af901574572da66fcc254cb35d0609c9080dd/pywin32-311-cp314-cp314-win_arm64.whl", hash = "sha256:a508e2d9025764a8270f93111a970e1d0fbfc33f4153b388bb649b7eec4f9b42", size = 8932540 }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766 }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075 }, +] + +[[package]] +name = "rich" +version = "14.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/67/cae617f1351490c25a4b8ac3b8b63a4dda609295d8222bad12242dfdc629/rich-14.3.4.tar.gz", hash = "sha256:817e02727f2b25b40ef56f5aa2217f400c8489f79ca8f46ea2b70dd5e14558a9", size = 230524 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/76/6d163cfac87b632216f71879e6b2cf17163f773ff59c00b5ff4900a80fa3/rich-14.3.4-py3-none-any.whl", hash = "sha256:07e7adb4690f68864777b1450859253bed81a99a31ac321ac1817b2313558952", size = 310480 }, +] + +[[package]] +name = "rpds-py" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4d/6e/f964e88b3d2abee2a82c1ac8366da848fce1c6d834dc2132c3fda3970290/rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425", size = 370157 }, + { url = "https://files.pythonhosted.org/packages/94/ba/24e5ebb7c1c82e74c4e4f33b2112a5573ddc703915b13a073737b59b86e0/rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d", size = 359676 }, + { url = "https://files.pythonhosted.org/packages/84/86/04dbba1b087227747d64d80c3b74df946b986c57af0a9f0c98726d4d7a3b/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4", size = 389938 }, + { url = "https://files.pythonhosted.org/packages/42/bb/1463f0b1722b7f45431bdd468301991d1328b16cffe0b1c2918eba2c4eee/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f", size = 402932 }, + { url = "https://files.pythonhosted.org/packages/99/ee/2520700a5c1f2d76631f948b0736cdf9b0acb25abd0ca8e889b5c62ac2e3/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4", size = 525830 }, + { url = "https://files.pythonhosted.org/packages/e0/ad/bd0331f740f5705cc555a5e17fdf334671262160270962e69a2bdef3bf76/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97", size = 412033 }, + { url = "https://files.pythonhosted.org/packages/f8/1e/372195d326549bb51f0ba0f2ecb9874579906b97e08880e7a65c3bef1a99/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89", size = 390828 }, + { url = "https://files.pythonhosted.org/packages/ab/2b/d88bb33294e3e0c76bc8f351a3721212713629ffca1700fa94979cb3eae8/rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d", size = 404683 }, + { url = "https://files.pythonhosted.org/packages/50/32/c759a8d42bcb5289c1fac697cd92f6fe01a018dd937e62ae77e0e7f15702/rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038", size = 421583 }, + { url = "https://files.pythonhosted.org/packages/2b/81/e729761dbd55ddf5d84ec4ff1f47857f4374b0f19bdabfcf929164da3e24/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7", size = 572496 }, + { url = "https://files.pythonhosted.org/packages/14/f6/69066a924c3557c9c30baa6ec3a0aa07526305684c6f86c696b08860726c/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed", size = 598669 }, + { url = "https://files.pythonhosted.org/packages/5f/48/905896b1eb8a05630d20333d1d8ffd162394127b74ce0b0784ae04498d32/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85", size = 561011 }, + { url = "https://files.pythonhosted.org/packages/22/16/cd3027c7e279d22e5eb431dd3c0fbc677bed58797fe7581e148f3f68818b/rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c", size = 221406 }, + { url = "https://files.pythonhosted.org/packages/fa/5b/e7b7aa136f28462b344e652ee010d4de26ee9fd16f1bfd5811f5153ccf89/rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825", size = 236024 }, + { url = "https://files.pythonhosted.org/packages/14/a6/364bba985e4c13658edb156640608f2c9e1d3ea3c81b27aa9d889fff0e31/rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229", size = 229069 }, + { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086 }, + { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053 }, + { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763 }, + { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951 }, + { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622 }, + { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492 }, + { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080 }, + { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680 }, + { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589 }, + { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289 }, + { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737 }, + { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120 }, + { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782 }, + { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463 }, + { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868 }, + { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887 }, + { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904 }, + { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945 }, + { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783 }, + { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021 }, + { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589 }, + { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025 }, + { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895 }, + { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799 }, + { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731 }, + { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027 }, + { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020 }, + { url = "https://files.pythonhosted.org/packages/0b/5d/47c4655e9bcd5ca907148535c10e7d489044243cc9941c16ed7cd53be91d/rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d", size = 223139 }, + { url = "https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15", size = 240224 }, + { url = "https://files.pythonhosted.org/packages/24/95/ffd128ed1146a153d928617b0ef673960130be0009c77d8fbf0abe306713/rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1", size = 230645 }, + { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443 }, + { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375 }, + { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850 }, + { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812 }, + { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841 }, + { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149 }, + { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843 }, + { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507 }, + { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949 }, + { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790 }, + { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217 }, + { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806 }, + { url = "https://files.pythonhosted.org/packages/f9/39/7e067bb06c31de48de3eb200f9fc7c58982a4d3db44b07e73963e10d3be9/rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6", size = 211341 }, + { url = "https://files.pythonhosted.org/packages/0a/4d/222ef0b46443cf4cf46764d9c630f3fe4abaa7245be9417e56e9f52b8f65/rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d", size = 225768 }, + { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099 }, + { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192 }, + { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080 }, + { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841 }, + { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670 }, + { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005 }, + { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112 }, + { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049 }, + { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661 }, + { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606 }, + { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126 }, + { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371 }, + { url = "https://files.pythonhosted.org/packages/72/c7/81dadd7b27c8ee391c132a6b192111ca58d866577ce2d9b0ca157552cce0/rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed", size = 215298 }, + { url = "https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950", size = 228604 }, + { url = "https://files.pythonhosted.org/packages/e8/95/ab005315818cc519ad074cb7784dae60d939163108bd2b394e60dc7b5461/rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6", size = 222391 }, + { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868 }, + { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747 }, + { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795 }, + { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330 }, + { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194 }, + { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340 }, + { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765 }, + { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834 }, + { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470 }, + { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630 }, + { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148 }, + { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030 }, + { url = "https://files.pythonhosted.org/packages/20/53/7c7e784abfa500a2b6b583b147ee4bb5a2b3747a9166bab52fec4b5b5e7d/rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0", size = 211570 }, + { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532 }, + { url = "https://files.pythonhosted.org/packages/69/71/3f34339ee70521864411f8b6992e7ab13ac30d8e4e3309e07c7361767d91/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58", size = 372292 }, + { url = "https://files.pythonhosted.org/packages/57/09/f183df9b8f2d66720d2ef71075c59f7e1b336bec7ee4c48f0a2b06857653/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a", size = 362128 }, + { url = "https://files.pythonhosted.org/packages/7a/68/5c2594e937253457342e078f0cc1ded3dd7b2ad59afdbf2d354869110a02/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb", size = 391542 }, + { url = "https://files.pythonhosted.org/packages/49/5c/31ef1afd70b4b4fbdb2800249f34c57c64beb687495b10aec0365f53dfc4/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c", size = 404004 }, + { url = "https://files.pythonhosted.org/packages/e3/63/0cfbea38d05756f3440ce6534d51a491d26176ac045e2707adc99bb6e60a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3", size = 527063 }, + { url = "https://files.pythonhosted.org/packages/42/e6/01e1f72a2456678b0f618fc9a1a13f882061690893c192fcad9f2926553a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5", size = 413099 }, + { url = "https://files.pythonhosted.org/packages/b8/25/8df56677f209003dcbb180765520c544525e3ef21ea72279c98b9aa7c7fb/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738", size = 392177 }, + { url = "https://files.pythonhosted.org/packages/4a/b4/0a771378c5f16f8115f796d1f437950158679bcd2a7c68cf251cfb00ed5b/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f", size = 406015 }, + { url = "https://files.pythonhosted.org/packages/36/d8/456dbba0af75049dc6f63ff295a2f92766b9d521fa00de67a2bd6427d57a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877", size = 423736 }, + { url = "https://files.pythonhosted.org/packages/13/64/b4d76f227d5c45a7e0b796c674fd81b0a6c4fbd48dc29271857d8219571c/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a", size = 573981 }, + { url = "https://files.pythonhosted.org/packages/20/91/092bacadeda3edf92bf743cc96a7be133e13a39cdbfd7b5082e7ab638406/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4", size = 599782 }, + { url = "https://files.pythonhosted.org/packages/d1/b7/b95708304cd49b7b6f82fdd039f1748b66ec2b21d6a45180910802f1abf1/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e", size = 562191 }, +] + +[[package]] +name = "s3transfer" +version = "0.17.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/ec/7c692cde9125b77e84b307354d4fb705f98b8ccad59a036d5957ca75bfc3/s3transfer-0.17.0.tar.gz", hash = "sha256:9edeb6d1c3c2f89d6050348548834ad8289610d886e5bf7b7207728bd43ce33a", size = 155337 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/72/c6c32d2b657fa3dad1de340254e14390b1e334ce38268b7ad51abda3c8c2/s3transfer-0.17.0-py3-none-any.whl", hash = "sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20", size = 86811 }, +] + +[[package]] +name = "sentry-sdk" +version = "2.60.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/54/a2/2e6c090db384cc515069f4f85542bd5baf6786852073020ea73d4a76d3ea/sentry_sdk-2.60.0.tar.gz", hash = "sha256:0bd25e54e78ca02d0be512529fa644bbbf9e8470d7b26371294012d4ca93c978", size = 452946 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/41/f2b800b7f12a05dd48c2a6280d4dd812d1425fc66ed3fe3fd99420c41d1a/sentry_sdk-2.60.0-py3-none-any.whl", hash = "sha256:28a536c03291c8bcb363cf35c611b32738ec118ff64d8d6383b096448ac4c803", size = 475616 }, +] + +[[package]] +name = "setuptools" +version = "82.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223 }, +] + +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755 }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050 }, +] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, +] + +[[package]] +name = "sse-starlette" +version = "3.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/2b/58abc2d1fd397e7dde08e947e05c884d8ef2f78d5e2588c17a12d42d6994/sse_starlette-3.4.4.tar.gz", hash = "sha256:07e0fa0460138baf25cdd5fb28683472c3995dc1642225191b3832d62526bcb0", size = 31819 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/67/805710444ea8cc75fbf70b920ed431a560c4bf9c57f7d5a3117213189399/sse_starlette-3.4.4-py3-none-any.whl", hash = "sha256:3f4dd50d8aed2771a091f3a83000323fc3844541c16b4fe585ae2420cc6df973", size = 16514 }, +] + +[[package]] +name = "starlette" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/66/4d20cdf39a8d6a51e663b7038e3b828ff211d3891a43a713fe7e4643f3a8/starlette-1.1.0.tar.gz", hash = "sha256:e83c7fe0ddecd8719c5b840080325aec0260acec86e9832899e377b91d65e90f", size = 2660060 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/79/920b8e0a8b20f793e8d64855095cb8febabf6175b8550b6f7a547d813891/starlette-1.1.0-py3-none-any.whl", hash = "sha256:7f0dfd38e428aad5cb6f9f667f0ca1d2d8ca3f3385dccac8305f79ec98458382", size = 72899 }, +] + +[[package]] +name = "tabulate" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/fe/802052aecb21e3797b8f7902564ab6ea0d60ff8ca23952079064155d1ae1/tabulate-0.9.0.tar.gz", hash = "sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c", size = 81090 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252 }, +] + +[[package]] +name = "tenacity" +version = "9.1.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/47/c6/ee486fd809e357697ee8a44d3d69222b344920433d3b6666ccd9b374630c/tenacity-9.1.4.tar.gz", hash = "sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a", size = 49413 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926 }, +] + +[[package]] +name = "teradata-mcp-evals" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "boto3" }, + { name = "deepeval" }, + { name = "mcp" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, + { name = "python-dotenv" }, + { name = "teradatasql" }, +] + +[package.metadata] +requires-dist = [ + { name = "boto3", specifier = ">=1.34.0" }, + { name = "deepeval", specifier = ">=1.4.0" }, + { name = "mcp", specifier = ">=1.0.0" }, + { name = "pytest", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", specifier = ">=0.23.0" }, + { name = "python-dotenv", specifier = ">=1.0.0" }, + { name = "teradatasql", specifier = ">=20.0.0" }, +] + +[[package]] +name = "teradatasql" +version = "20.0.0.59" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycryptodome" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/2f/ad23eeef90eaea7c3cff3f0a5508bc7389fe3678eaf3e3616e7dd9a87e6f/teradatasql-20.0.0.59-py3-none-any.whl", hash = "sha256:971ff04a9664f3a08fcc2431e5c1d789d63654859f1856cd7e6328c30224c0cc", size = 210821129 }, +] + +[[package]] +name = "tqdm" +version = "4.67.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374 }, +] + +[[package]] +name = "typer" +version = "0.25.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "click" }, + { name = "rich" }, + { name = "shellingham" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/51/9aed62104cea109b820bbd6c14245af756112017d309da813ef107d42e7e/typer-0.25.1.tar.gz", hash = "sha256:9616eb8853a09ffeabab1698952f33c6f29ffdbceb4eaeecf571880e8d7664cc", size = 122276 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/f9/2b3ff4e56e5fa7debfaf9eb135d0da96f3e9a1d5b27222223c7296336e5f/typer-0.25.1-py3-none-any.whl", hash = "sha256:75caa44ed46a03fb2dab8808753ffacdbfea88495e74c85a28c5eefcf5f39c89", size = 58409 }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614 }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611 }, +] + +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087 }, +] + +[[package]] +name = "uvicorn" +version = "0.47.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f6/b1/8e7077a8641086aea449e1b5752a570f1b5906c64e0a33cd6d93b63a066b/uvicorn-0.47.0.tar.gz", hash = "sha256:7c9a0ea1a9414106bbab7324609c162d8fa0cdcdcb703060987269d77c7bb533", size = 90582 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/41/ac2dfdbc1f60c7af4f994c7a335cfa7040c01642b605d65f611cecc2a1e4/uvicorn-0.47.0-py3-none-any.whl", hash = "sha256:2c5715bc12d1892d84752049f400cd1c3cb018514967fdfeb97640443a6a9432", size = 71301 }, +] + +[[package]] +name = "wheel" +version = "0.47.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/62/75f18a0f03b4219c456652c7780e4d749b929eb605c098ce3a5b6b6bc081/wheel-0.47.0.tar.gz", hash = "sha256:cc72bd1009ba0cf63922e28f94d9d83b920aa2bb28f798a31d0691b02fa3c9b3", size = 63854 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/1b/9e33c09813d65e248f7f773119148a612516a4bea93e9c6f545f78455b7c/wheel-0.47.0-py3-none-any.whl", hash = "sha256:212281cab4dff978f6cedd499cd893e1f620791ca6ff7107cf270781e587eced", size = 32218 }, +] + +[[package]] +name = "yarl" +version = "1.24.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "multidict" }, + { name = "propcache" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/12/1e8f37460ea0f7eb59c221fdaf0ed75e7ac43e97f8093b9c6f411df50a78/yarl-1.24.2.tar.gz", hash = "sha256:9ac374123c6fd7abf64d1fec93962b0bd4ee2c19751755a762a72dd96c0378f8", size = 210798 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c5/c5/1ce244152ff2839645e7cae92f90e7bafcb2c52bea7ff586ac714f14f5df/yarl-1.24.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:36348bebb147b83818b9d7e673ea4debc75970afc6ffdc7e3975ad05ce5a58c1", size = 128971 }, + { url = "https://files.pythonhosted.org/packages/87/5a/00f36967203ed89cb3acd2c8ed526cc3fed9418eb70ce128160a911c8499/yarl-1.24.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1a97e42c8a2233f2f279ecadd9e4a037bcb5d813b78435e8eedd4db5a9e9708c", size = 91507 }, + { url = "https://files.pythonhosted.org/packages/31/d0/1fb0c1cd27288f39f6974da4318c32768d72c9890984541fdf1e2e32a51d/yarl-1.24.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8d027d56f1035e339d1001ac33eceab5b2ec8e42e449787bb75e289fb9a5cd1d", size = 91343 }, + { url = "https://files.pythonhosted.org/packages/03/ce/d4a646508bed2f8dec6435b40166fe9308dd191262033d3f307b2bbcaecd/yarl-1.24.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a6377060e7927187a42b7eb202090cbe2b34933a4eeaf90e3bd9e33432e5cae", size = 105704 }, + { url = "https://files.pythonhosted.org/packages/4b/07/b3278e82d8bc41485bcf6d856cd0433262593de615b1d3dc43bd3f5bead4/yarl-1.24.2-cp311-cp311-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:17076578bce0049a5ce57d14ad1bded391b68a3b213e9b81b0097b090244999a", size = 97281 }, + { url = "https://files.pythonhosted.org/packages/17/5b/4cee6e7c92e487bebe7afc797da0aa54a248ab4e776a68fe369ec29665a5/yarl-1.24.2-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:50713f1d4d6be6375bb178bb43d140ee1acb8abe589cd723320b7925a275be1e", size = 114020 }, + { url = "https://files.pythonhosted.org/packages/5c/82/111076571545a7d4f9cca3fbd5c6f40615af58642be09f12328f48022468/yarl-1.24.2-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:34263e2fa8fb5bb63a0d97706cda38edbad62fddb58c7f12d6acbc092812aa50", size = 111450 }, + { url = "https://files.pythonhosted.org/packages/b6/ec/08f671f69a444d704aeecebf92af659b67b97a869942411d0a578b08c334/yarl-1.24.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:49016d82f032b1bd1e10b01078a7d29ae71bf468eeae0ea22df8bab691e60003", size = 106384 }, + { url = "https://files.pythonhosted.org/packages/e5/86/ce41e7a7a199340b2330d52b60f25c4074b6636dd0e60b1a80d31a9db042/yarl-1.24.2-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:3f6d2c216318f8f32038ca3f72501ba08536f0fd18a36e858836b121b2deed9f", size = 106153 }, + { url = "https://files.pythonhosted.org/packages/c4/5d/31be8a729531ab3e55ac3e7e5c800be8c89ea98947f418b2f6ea259fb6ee/yarl-1.24.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:08d3a33218e0c64393e7610284e770409a9c31c429b078bcb24096ed0a783b8f", size = 105322 }, + { url = "https://files.pythonhosted.org/packages/47/9b/b57afb22b386ae87ac9940f09878b98d8c333f89113e6fc96fcf4ca9eb64/yarl-1.24.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5d699376c4ca3cba49bbfae3a05b5b70ded572937171ce1e0b8d87118e2ba294", size = 99057 }, + { url = "https://files.pythonhosted.org/packages/a3/4f/06348c27c8389256c313e8a57d796808fc0264c915dd5e7cfd3c0e314dc7/yarl-1.24.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:a1cab588b4fa14bea2e55ebea27478adfb05372f47573738e1acc4a36c0b05d2", size = 113502 }, + { url = "https://files.pythonhosted.org/packages/5f/1c/284f307b298e4a17b7943b07d9d7ecc4151537f8d137ba51f3bb6c31ca20/yarl-1.24.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:ec87ccc31bd21db7ad009d8572c127c1000f268517618a4cc09adba3c2a7f21c", size = 105253 }, + { url = "https://files.pythonhosted.org/packages/c8/bf/0de123bec8619e45c80cbded9085f61b5b4a9eddb8abe6d25d28ee1ec866/yarl-1.24.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:d1dd47a22843b212baa8d74f37796815d43bd046b42a0f41e9da433386c3136b", size = 111345 }, + { url = "https://files.pythonhosted.org/packages/90/af/0248eb065e51129d2a9b2436cd1b5c772c19a6b04e5b6a186955671e3319/yarl-1.24.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7b54b9c67c2b06bd7b9a77253d242124b9c95d2c02def5a1144001ee547dd9d5", size = 106558 }, + { url = "https://files.pythonhosted.org/packages/21/3c/f960d7a65ef97d8ba9b424fb5128796a4bc710fc6df2ddbbd7dfdc3bbd20/yarl-1.24.2-cp311-cp311-win_amd64.whl", hash = "sha256:f8fdbcff8b2c7c9284e60c196f693588598ddcee31e11c18e14949ce44519d45", size = 92808 }, + { url = "https://files.pythonhosted.org/packages/03/1a/49fb03750e4de4d2284cd5b885a383133c34eef45bd59631b2bb8b7e81e8/yarl-1.24.2-cp311-cp311-win_arm64.whl", hash = "sha256:b32c37a7a337e90822c45797bf3d79d60875cfcccd3ecc80e9f453d87026c122", size = 87610 }, + { url = "https://files.pythonhosted.org/packages/f0/da/866bcb01076ba49d2b42b309867bed3826421f1c479655eb7a607b44f20b/yarl-1.24.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:b975866c184564c827e0877380f0dae57dcca7e52782128381b72feff6dfceb8", size = 129957 }, + { url = "https://files.pythonhosted.org/packages/bf/1d/fcefb70922ea2268a8971d8e5874d9a8218644200fb8465f1dcad55e6851/yarl-1.24.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3b075301a2836a0e297b1b658cb6d6135df535d62efefdd60366bd589c2c82f2", size = 92164 }, + { url = "https://files.pythonhosted.org/packages/29/b6/170e2b8d4e3bc30e6bfdcca53556537f5bf595e938632dfcb059311f3ff6/yarl-1.24.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8ae44649b00947634ab0dab2a374a638f52923a6e67083f2c156cd5cbd1a881d", size = 91688 }, + { url = "https://files.pythonhosted.org/packages/fe/a5/c9f655d5553ea0b99fdac9d6a99ad3f9b3e73b8e5758bb46f58c9831f74c/yarl-1.24.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:507cc19f0b45454e2d6dcd62ff7d062b9f77a2812404e62dbdaec05b50faa035", size = 102902 }, + { url = "https://files.pythonhosted.org/packages/5d/bc/6b9664d815d79af4ee553337f9d606c56bbf269186ada9172de45f1b5f60/yarl-1.24.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c4c17bad5a530912d2111825d3f05e89bab2dd376aaa8cbc77e449e6db63e576", size = 97931 }, + { url = "https://files.pythonhosted.org/packages/98/ec/32ba48acae30fecd60928f5791188b80a9d6ee3840507ffda29fecd37b71/yarl-1.24.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:f5f0cbb112838a4a293985b6ed73948a547dadcc1ba6d2089938e7abdedceef8", size = 111030 }, + { url = "https://files.pythonhosted.org/packages/82/5a/6f4cd081e5f4934d2ae3a8ef4abe3afacc010d26f0035ee91b35cd7d7c37/yarl-1.24.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ec8356b8a6afcf81fc7aeeef13b1ff7a49dec00f313394bbb9e83830d32ccd7", size = 110392 }, + { url = "https://files.pythonhosted.org/packages/7a/da/323a01c349bd5fb01bb6652e314d9bb218cee630a736bdb810ad50e4013f/yarl-1.24.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7e7ebcdef69dec6c6451e616f32b622a6d4a2e92b445c992f7c8e5274a6bbc4c", size = 105612 }, + { url = "https://files.pythonhosted.org/packages/7c/80/264ab684f181e1a876389374519ff05d10248725535ae2ac4e8ac4e563d6/yarl-1.24.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:47a55d6cf6db2f401017a9e96e5288844e5051911fb4e0c8311a3980f5e59a7d", size = 104487 }, + { url = "https://files.pythonhosted.org/packages/41/07/efabe5df87e96d7ad5959760b888344be48cd6884db127b407c6b5503adc/yarl-1.24.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3065657c80a2321225e804048597ad55658a7e76b32d6f5ee4074d04c50401db", size = 102333 }, + { url = "https://files.pythonhosted.org/packages/44/0c/bcf7c42603e1009295f586d8890f2ba032c8b53310e815adf0a202c73d9f/yarl-1.24.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:cb84b80d88e19ede158619b80813968713d8d008b0e2497a576e6a0557d50712", size = 99025 }, + { url = "https://files.pythonhosted.org/packages/4f/82/84482ab1a57a0f21a08afe6a7004c61d741f8f2ecc3b05c321577c612164/yarl-1.24.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:990de4f680b1c217e77ff0d6aa0029f9eb79889c11fb3e9a3942c7eba29c1996", size = 110507 }, + { url = "https://files.pythonhosted.org/packages/c4/8d/a546ba1dfe1b0f290e05fef145cd07614c0f15df1a707195e512d1e39d1d/yarl-1.24.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:abb8ec0323b80161e3802da3150ef660b41d0e9be2048b76a363d93eee992c2b", size = 103719 }, + { url = "https://files.pythonhosted.org/packages/1a/b6/267f2a09213138473adfce6b8a6e17791d7fee70bd4d9003218e4dec58b0/yarl-1.24.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:e7977781f83638a4c73e0f88425563d70173e0dfd90ac006a45c65036293ee3c", size = 110438 }, + { url = "https://files.pythonhosted.org/packages/48/2d/1c8d89c7c5f9cad9fb2902445d94e2ab1d7aa35de029afbb8ae95c42d00f/yarl-1.24.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e30dd55825dc554ec5b66a94953b8eda8745926514c5089dfcacecb9c99b5bd1", size = 105719 }, + { url = "https://files.pythonhosted.org/packages/a7/25/722e3b93bd687009afb2d59a35e13d30ddd8f80571445bb0c4e4ce26ec66/yarl-1.24.2-cp312-cp312-win_amd64.whl", hash = "sha256:7dafe10c12ddd4d120d528c4b5599c953bd7b12845347d507b95451195bb6cad", size = 92901 }, + { url = "https://files.pythonhosted.org/packages/39/47/4486ccfb674c04854a1ef8aa77868b6a6f765feaf69633409d7ca4f02cb8/yarl-1.24.2-cp312-cp312-win_arm64.whl", hash = "sha256:044a09d8401fcf8681977faef6d286b8ade1e2d2e9dceda175d1cfa5ca496f30", size = 87229 }, + { url = "https://files.pythonhosted.org/packages/82/62/fcf0ce677f17e5c471c06311dd25964be38a4c586993632910d2e75278bc/yarl-1.24.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:491ac9141decf49ee8030199e1ee251cdff0e131f25678817ff6aa5f837a3536", size = 128978 }, + { url = "https://files.pythonhosted.org/packages/d3/58/8e63299bb71ed61a834121d9d3fe6c9fcf2a6a5d09754ff4f20f2d20baf5/yarl-1.24.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e89418f65eda18f99030386305bd44d7d504e328a7945db1ead514fbe03a0607", size = 91733 }, + { url = "https://files.pythonhosted.org/packages/c1/24/16748d5dab6daec8b0ed81ccec639a1cded0f18dcc62a4f696b4fe366c37/yarl-1.24.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cdfcce633b4a4bb8281913c57fcafd4b5933fbc19111a5e3930bbd299d6102f1", size = 91113 }, + { url = "https://files.pythonhosted.org/packages/1b/66/b63fff7b71211e866624b21432d5943cbb633eb0c2872d9ee3070648f22c/yarl-1.24.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:863297ddede92ee49024e9a9b11ecb59f310ca85b60d8537f56bed9bbb5b1986", size = 103899 }, + { url = "https://files.pythonhosted.org/packages/9d/ac/ba1974b8533909636f7733fe86cf677e3619527c3c2fa913e0ea89c48757/yarl-1.24.2-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:374423f70754a2c96942ede36a29d37dc6b0cb8f92f8d009ddf3ed78d3da5488", size = 97862 }, + { url = "https://files.pythonhosted.org/packages/1b/a5/123ac993b5c2ba6f554a140305620cb8f150fa543711bbc49be3ec0a65a4/yarl-1.24.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:33a29b5d00ccbf3219bb3e351d7875739c19481e030779f48cc46a7a71681a9b", size = 111060 }, + { url = "https://files.pythonhosted.org/packages/23/37/c472d3af3509688392134a88a825276770a187f1daa4de3f6dc0a327a751/yarl-1.24.2-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a9532c57211730c515341af11fef6e9b61d157487272a096d0c04da445642592", size = 110613 }, + { url = "https://files.pythonhosted.org/packages/df/88/09c28dad91e662ccfaa1b78f1c57badde74fc9d0b23e74aef644750ecd73/yarl-1.24.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:91e72cf093fd833483a97ee648e0c053c7c629f51ff4a0e7edd84f806b0c5617", size = 107012 }, + { url = "https://files.pythonhosted.org/packages/07/ab/9d4f69d571a94f4d112fa7e2e007200f5a54d319f58c82ac7b7baa61f5c6/yarl-1.24.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b3177bc0a768ef3bacceb4f272632990b7bea352f1b2f1eee9d6d6ff16516f92", size = 105887 }, + { url = "https://files.pythonhosted.org/packages/8e/9a/000b2b66c0d772a499fc531d21dab92dfeb73b640a12eed6ba89f49bb2d0/yarl-1.24.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e196952aacaf3b232e265ff02980b64d483dc0972bd49bcb061171ff22ac203a", size = 103620 }, + { url = "https://files.pythonhosted.org/packages/41/7c/7c1050f73450fbdaa3f0c72017059f00ce5e13366692f3dba25275a1083d/yarl-1.24.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:204e7a61ce99919c0de1bf904ab5d7aa188a129ea8f690a8f76cfb6e2844dc44", size = 100599 }, + { url = "https://files.pythonhosted.org/packages/ec/b1/29e5756b3926705f5f6089bd5b9f50a56eaac550da6e260bf713ead44d04/yarl-1.24.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4b156914620f0b9d78dc1adb3751141daee561cfec796088abb89ed49d220f1a", size = 110604 }, + { url = "https://files.pythonhosted.org/packages/a3/4b/8415bc96e9b150cde942fbac9a8182985e58f40ce5c54c34ed015407d3ee/yarl-1.24.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:8372a2b976cf70654b2be6619ab6068acabb35f724c0fda7b277fbf53d66a5cf", size = 105161 }, + { url = "https://files.pythonhosted.org/packages/8b/d4/cde059abfa229553b7298a2eadde2752e723d50aeedaef86ce59da2718ee/yarl-1.24.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:f9a1e9b622ca284143aab5d885848686dcd85453bb1ca9abcdb7503e64dc0056", size = 110619 }, + { url = "https://files.pythonhosted.org/packages/e7/2c/d6a6c9a61549f7b6c7e6dc6937d195bcf069582b47b7200dcd0e7b256acf/yarl-1.24.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:810e19b685c8c3c5862f6a38160a1f4e4c0916c9390024ec347b6157a45a0992", size = 107362 }, + { url = "https://files.pythonhosted.org/packages/92/dd/3ae5fe417e9d1c353a548553326eb9935e76b6b727161563b424cc296df3/yarl-1.24.2-cp313-cp313-win_amd64.whl", hash = "sha256:7d37fb7c38f2b6edab0f845c4f85148d4c44204f52bc127021bd2bc9fdbf1656", size = 92667 }, + { url = "https://files.pythonhosted.org/packages/10/cc/a7beb239f78f27fca1b053c8e8595e4179c02e62249b4687ec218c370c50/yarl-1.24.2-cp313-cp313-win_arm64.whl", hash = "sha256:1e831894be7c2954240e49791fa4b50c05a0dc881de2552cfe3ffd8631c7f461", size = 87069 }, + { url = "https://files.pythonhosted.org/packages/40/0e/e08087695fc12789263821c5dc0f8dc52b5b17efd0887cacf419f8a43ba3/yarl-1.24.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:f9312b3c02d9b3d23840f67952913c9c8721d7f1b7db305289faefa878f364c2", size = 129670 }, + { url = "https://files.pythonhosted.org/packages/3a/98/ab4b5ed1b1b5cd973c8a3eb994c3a6aefb6ce6d399e21bb5f0316c33815c/yarl-1.24.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a4f4d6cd615823bfc7fb7e9b5987c3f41666371d870d51058f77e2680fbe9630", size = 91916 }, + { url = "https://files.pythonhosted.org/packages/ba/b1/5297bb6a7df4782f7605bffc43b31f5044070935fbbcaa6c705a07e6ac65/yarl-1.24.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0c3063e5c0a8e8e62fae6c2596fa01da1561e4cd1da6fec5789f5cf99a8aefd8", size = 91625 }, + { url = "https://files.pythonhosted.org/packages/02/a7/45baabfff76829264e623b185cff0c340d7e11bf3e1cd9ea37e7d17934bd/yarl-1.24.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fecd17873a096036c1c87ab3486f1aef7f269ada7f23f7f856f93b1cc7744f14", size = 104574 }, + { url = "https://files.pythonhosted.org/packages/f3/40/3a5ab144d3d650ca37d4f4b57e56169be8af3ca34c448793e064b30baaed/yarl-1.24.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:a46d1ab4ba4d32e6dc80daf8a28ce0bd83d08df52fbc32f3e288663427734535", size = 97534 }, + { url = "https://files.pythonhosted.org/packages/9c/b5/5658fef3681fb5776b4513b052bec750009f47b3a592251c705d75375798/yarl-1.24.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:73e68edf6dfd5f73f9ca127d84e2a6f9213c65bdffb736bda19524c0564fcd14", size = 111481 }, + { url = "https://files.pythonhosted.org/packages/4c/06/fdcd7dde037f00866dce123ed4ba23dba94beb56fc4cf561668d27be37f2/yarl-1.24.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a296ca617f2d25fbceafb962b88750d627e5984e75732c712154d058ae8d79a3", size = 111529 }, + { url = "https://files.pythonhosted.org/packages/c2/53/d81269aaafccea0d33396c03035de997b743f11e648e6e27a0df99c72980/yarl-1.24.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e51b2cf5ec89a8b8470177641ed62a3ba22d74e1e898e06ad53aa77972487208", size = 107338 }, + { url = "https://files.pythonhosted.org/packages/ae/04/23049463f729bd899df203a7960505a75333edd499cda8aa1d5a82b64df5/yarl-1.24.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:310fc687f7b2044ec54e372c8cbe923bb88f5c37bded0d3079e5791c2fc3cf50", size = 106147 }, + { url = "https://files.pythonhosted.org/packages/14/18/04a4b5830b43ed5e4c5015b40e9f6241ad91487d71611061b4e111d6ac80/yarl-1.24.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:297a2fe352ecf858b30a98f87948746ec16f001d279f84aebdbd3bd965e2f1bd", size = 104272 }, + { url = "https://files.pythonhosted.org/packages/5a/f7/8cffdf319aee7a7c1dbd07b61d91c3e3fda460c7a93b5f93e445f3806c4c/yarl-1.24.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:2a263e76b97bc42bdcd7c5f4953dec1f7cd62a1112fa7f869e57255229390d67", size = 99962 }, + { url = "https://files.pythonhosted.org/packages/d7/39/b3cce3b7dbef64ac700ad4cea156a207d01bede0f507587616c364b5468e/yarl-1.24.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:822519b64cf0b474f1a0aaef1dc621438ea46bb77c94df97a5b4d213a7d8a8b1", size = 111063 }, + { url = "https://files.pythonhosted.org/packages/a1/ea/100818505e7ebf165c7242ff17fdf7d9fee79e27234aeca871c1082920d7/yarl-1.24.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:b6067060d9dc594899ba83e6db6c48c68d1e494a6dab158156ed86977ca7bcb1", size = 105438 }, + { url = "https://files.pythonhosted.org/packages/8f/d2/e075a0b32aa6625087de9e653087df0759fed5de4a435fef594181102a77/yarl-1.24.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:0063adad533e57171b79db3943b229d40dfafeeee579767f96541f106bac5f1b", size = 111458 }, + { url = "https://files.pythonhosted.org/packages/e6/5c/ceea7ba98b65c8eb8d947fdc52f9bedfcd43c6a57c9e3c90c17be8f324a3/yarl-1.24.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ee8e3fb34513e8dc082b586ef4910c98335d43a6fab688cd44d4851bacfce3e8", size = 107589 }, + { url = "https://files.pythonhosted.org/packages/fa/d9/5582d57e2b2db9b85eb6663a22efdd78e08805f3f5389566e9fcad254d1b/yarl-1.24.2-cp314-cp314-win_amd64.whl", hash = "sha256:afb00d7fd8e0f285ca29a44cc50df2d622ff2f7a6d933fa641577b5f9d5f3db0", size = 94424 }, + { url = "https://files.pythonhosted.org/packages/92/10/7dc07a0e22806a9280f42a57361395506e800c64e22737cd7b0886feab42/yarl-1.24.2-cp314-cp314-win_arm64.whl", hash = "sha256:68cf6eacd6028ef1142bc4b48376b81566385ca6f9e7dde3b0fa91be08ffcb57", size = 88690 }, + { url = "https://files.pythonhosted.org/packages/9e/13/d5b8e2c8667db955bcb3de233f18798fefe7edf1d7429c2c9d4f9c401114/yarl-1.24.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:221ce1dd921ac4f603957f17d7c18c5cc0797fbb52f156941f92e04605d1d67b", size = 136248 }, + { url = "https://files.pythonhosted.org/packages/de/46/a4a97c05c9c9b8fd266bb2a0df12992c7fbd02391eb9640583411b6dab32/yarl-1.24.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:5f3224db28173a00d7afacdee07045cc4673dfab2b15492c7ae10deddbece761", size = 95084 }, + { url = "https://files.pythonhosted.org/packages/95/b2/845cf2074a015e6fe0d0808cf1a2d9e868386c4220d657ebd8302b199043/yarl-1.24.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c557165320d6244ebe3a02431b2a201a20080e02f41f0cfa0ccc47a183765da8", size = 95272 }, + { url = "https://files.pythonhosted.org/packages/fe/16/e69d4aa244aef45235ddfebc0e04036a6829842bc5a6a795aedc6c998d23/yarl-1.24.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:904065e6e85b1fa54d0d87438bd58c14c0bad97aad654ad1077fd9d87e8478ed", size = 101497 }, + { url = "https://files.pythonhosted.org/packages/15/94/c07107715d621076863ee88b3ddf183fa5e9d4aba5769623c9979828410a/yarl-1.24.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:8cec2a38d70edc10e0e856ceda886af5327a017ccbde8e1de1bd44d300357543", size = 94002 }, + { url = "https://files.pythonhosted.org/packages/a9/35/fc1bbdd895b5e4010b8fdd037f7ed3aa289d3863e08231b30231ca9a0815/yarl-1.24.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e7484b9361ed222ee1ca5b4337aa4cbdcc4618ce5aff57d9ef1582fd95893fc0", size = 106524 }, + { url = "https://files.pythonhosted.org/packages/1f/f2/32b66d0a4ba47c296cf86d03e2c67bff58399fe6d6d84d5205c04c66cc6d/yarl-1.24.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:84f9670b89f34db07f81e53aee83e0b938a3412329d51c8f922488be7fcc4024", size = 106165 }, + { url = "https://files.pythonhosted.org/packages/95/47/37cb5ff50c5e825d4d38e81bb04d1b7e96bf960f7ab89f9850b162f3f114/yarl-1.24.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:abb2759733d63a28b4956500a5dd57140f26486c92b2caedfb964ab7d9b79dbf", size = 103010 }, + { url = "https://files.pythonhosted.org/packages/6f/d2/4597912315096f7bb359e46e13bf8b60994fcbb2db29b804c0902ef4eff5/yarl-1.24.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:081c2bf54efe03774d0311172bc04fedf9ca01e644d4cd8c805688e527209bdc", size = 101128 }, + { url = "https://files.pythonhosted.org/packages/b9/d5/c8e86e120521e646013d02a8e3b8884392e28494be8f392366e50d208efc/yarl-1.24.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:86746bef442aa479107fe28132e1277237f9c24c2f00b0b0cf22b3ee0904f2bb", size = 101382 }, + { url = "https://files.pythonhosted.org/packages/fa/98/70b229236118f89dbeb739b76f10225bbf53b5497725502594c9a01d699a/yarl-1.24.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:2d07d21d0bc4b17558e8de0b02fbfdf1e347d3bb3699edd00bb92e7c57925420", size = 95964 }, + { url = "https://files.pythonhosted.org/packages/87/f8/56c386981e3c8648d279fdef2397ffec577e8320fd5649745e34d54faeb7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:4fb1ac3fc5fecd8ae7453ea237e4d22b49befa70266dfe1629924245c21a0c7f", size = 106204 }, + { url = "https://files.pythonhosted.org/packages/1a/1e/765afe97811ca35933e2a7de70ac57b1997ea2e4ee895719ee7a231fb7e5/yarl-1.24.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:4da31a5512ed1729ca8d8aacde3f7faeb8843cde3165d6bcf7f88f74f17bb8aa", size = 101510 }, + { url = "https://files.pythonhosted.org/packages/ee/78/393913f4b9039e1edd09ae8a9bbb9d539be909a8abf6d8a2084585bed4b7/yarl-1.24.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:533ded4dceb5f1f3da7906244f4e82cf46cfd40d84c69a1faf5ac506aa65ecbe", size = 105584 }, + { url = "https://files.pythonhosted.org/packages/78/87/deb17b7049bbe74ea11a713b86f8f27800cc1c8648b0b797243ebb4830ba/yarl-1.24.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:7b3a85525f6e7eeabcfdd372862b21ee1915db1b498a04e8bf0e389b607ff0bd", size = 103410 }, + { url = "https://files.pythonhosted.org/packages/8f/be/f9f7594e23b5b93affff0318e4593c1920331bcaefda326cabcad94296a1/yarl-1.24.2-cp314-cp314t-win_amd64.whl", hash = "sha256:a7624b1ca46ca5d7b864ef0d2f8efe3091454085ee1855b4e992314529972215", size = 102980 }, + { url = "https://files.pythonhosted.org/packages/65/a4/ba80dccd3593ff1f01051a818694d07b58cb8232677ee9a22a5a1f93a9fc/yarl-1.24.2-cp314-cp314t-win_arm64.whl", hash = "sha256:e434a45ce2e7a947f951fc5a8944c8cc080b7e59f9c50ae80fd39107cf88126d", size = 91219 }, + { url = "https://files.pythonhosted.org/packages/fd/4d/4b880086bd0d3e034d25647be1d830afc3e3f610e98c4ab3490af6b1b6d5/yarl-1.24.2-py3-none-any.whl", hash = "sha256:2783d9226db8797636cd6896e4de81feed252d1db72265686c9558d97a4d94b9", size = 53576 }, +] From 315ba229c213da83ce70d8f826cdaa0dfc8b4037 Mon Sep 17 00:00:00 2001 From: dtehan-td Date: Tue, 23 Jun 2026 17:10:40 -0700 Subject: [PATCH 5/6] Fix asyncio.run() conflict in pytest context: use ThreadPoolExecutor when event loop already running --- tests/eval_test/agent/client.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tests/eval_test/agent/client.py b/tests/eval_test/agent/client.py index 9d93fed..dd982c3 100644 --- a/tests/eval_test/agent/client.py +++ b/tests/eval_test/agent/client.py @@ -122,6 +122,22 @@ def _make_bedrock_client(region: str = "us-east-1"): return boto3.client("bedrock-runtime", region_name=region) +def _run_async(coro): + """Run an async coroutine, handling both raw asyncio and pytest event loop contexts.""" + try: + loop = asyncio.get_running_loop() + except RuntimeError: + # No running loop — safe to use asyncio.run() + return asyncio.run(coro) + + # A loop is already running. This can happen in pytest with event loop plugins. + # Use asyncio.ensure_future and wait for it to complete. + import concurrent.futures + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(asyncio.run, coro) + return future.result() + + def _mcp_tool_to_bedrock(tool) -> dict: return { "toolSpec": { @@ -328,7 +344,7 @@ def run_agent( region=os.environ.get("AWS_REGION", "us-east-1"), ) - return asyncio.run( + return _run_async( _run_agent_async( prompt=prompt, model_id=resolved_model, @@ -361,7 +377,7 @@ def run_agent_turns( region=os.environ.get("AWS_REGION", "us-east-1"), ) - return asyncio.run( + return _run_async( _run_agent_turns_async( prompts=prompts, model_id=resolved_model, From e59629c8a3a26ac43be7cbaf89b1d347b491471b Mon Sep 17 00:00:00 2001 From: dtehan-td Date: Tue, 23 Jun 2026 17:31:47 -0700 Subject: [PATCH 6/6] chore: clean up eval results and update gitignore - Remove eval test results directory (large transient files) - Add results/, .mypy_cache/, .deepeval/, and .pytest_cache/ to gitignore - These are build artifacts and should not be tracked --- .gitignore | 5 + tests/eval_test/results/index.json | 46 - tests/eval_test/results/latest.json | 15 - tests/eval_test/results/latest_summary.json | 683 ---- tests/eval_test/results/latest_summary.md | 867 ----- .../manifest.json | 22 - .../summary.json | 3209 ----------------- .../summary.md | 268 -- .../manifest.json | 22 - .../summary.json | 683 ---- .../summary.md | 867 ----- .../manifest.json | 22 - .../summary.json | 683 ---- .../summary.md | 867 ----- 14 files changed, 5 insertions(+), 8254 deletions(-) delete mode 100644 tests/eval_test/results/index.json delete mode 100644 tests/eval_test/results/latest.json delete mode 100644 tests/eval_test/results/latest_summary.json delete mode 100644 tests/eval_test/results/latest_summary.md delete mode 100644 tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/manifest.json delete mode 100644 tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.json delete mode 100644 tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.md delete mode 100644 tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/manifest.json delete mode 100644 tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.json delete mode 100644 tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.md delete mode 100644 tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/manifest.json delete mode 100644 tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json delete mode 100644 tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md diff --git a/.gitignore b/.gitignore index 8b262b0..1dbda65 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ __pycache__/ logs/ working/ +results/ +.mypy_cache/ .env .DS_Store .vscode/ @@ -18,4 +20,7 @@ test_*.py .planning/ .ruff_cache/ Start_MCP_Server.bat +.deepeval/ +.pytest_cache/ + diff --git a/tests/eval_test/results/index.json b/tests/eval_test/results/index.json deleted file mode 100644 index 05ae1dc..0000000 --- a/tests/eval_test/results/index.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "runs": [ - { - "run_id": "2026-06-24T00-06-59.222908Z__base__baseline", - "started_at": "2026-06-24T00:06:59.222908+00:00", - "module_filter": "base", - "case_type_filter": "all", - "run_label": null, - "description_mode": "mcp_server", - "passed": 0, - "failed": 32, - "total": 32, - "run_dir": "runs/2026-06-24T00-06-59.222908Z__base__baseline", - "summary_json": "runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json", - "summary_md": "runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md" - }, - { - "run_id": "2026-06-24T00-05-13.720459Z__base__baseline", - "started_at": "2026-06-24T00:05:13.720459+00:00", - "module_filter": "base", - "case_type_filter": "all", - "run_label": null, - "description_mode": "mcp_server", - "passed": 0, - "failed": 32, - "total": 32, - "run_dir": "runs/2026-06-24T00-05-13.720459Z__base__baseline", - "summary_json": "runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.json", - "summary_md": "runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.md" - }, - { - "run_id": "2026-06-23T21-14-19.042368Z__all__baseline", - "started_at": "2026-06-23T21:14:19.042368+00:00", - "module_filter": "all", - "case_type_filter": "all", - "run_label": null, - "description_mode": "mcp_server", - "passed": 103, - "failed": 3, - "total": 106, - "run_dir": "runs/2026-06-23T21-14-19.042368Z__all__baseline", - "summary_json": "runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.json", - "summary_md": "runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.md" - } - ] -} diff --git a/tests/eval_test/results/latest.json b/tests/eval_test/results/latest.json deleted file mode 100644 index 88f35c5..0000000 --- a/tests/eval_test/results/latest.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "run_id": "2026-06-24T00-06-59.222908Z__base__baseline", - "started_at": "2026-06-24T00:06:59.222908+00:00", - "run_dir": "runs/2026-06-24T00-06-59.222908Z__base__baseline", - "summary_md": "runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md", - "summary_json": "runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json", - "manifest": "runs/2026-06-24T00-06-59.222908Z__base__baseline/manifest.json", - "module_filter": "base", - "case_type_filter": "all", - "run_label": null, - "description_mode": "mcp_server", - "passed": 0, - "failed": 32, - "total": 32 -} diff --git a/tests/eval_test/results/latest_summary.json b/tests/eval_test/results/latest_summary.json deleted file mode 100644 index 90bd6d6..0000000 --- a/tests/eval_test/results/latest_summary.json +++ /dev/null @@ -1,683 +0,0 @@ -{ - "run_id": "2026-06-24T00-06-59.222908Z__base__baseline", - "started_at": "2026-06-24T00:06:59.222908+00:00", - "module_filter": "base", - "case_type_filter": "all", - "run_label": null, - "agent_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "judge_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "evals_database": "demo_user", - "description_mode": "mcp_server", - "description_overrides_file": null, - "description_override_count": 0, - "total": 32, - "passed": 0, - "failed": 32, - "cases": [ - { - "case_id": "base_readQuery_happy", - "case_type": "happy_path", - "description": "Execute a SQL query against the evals employees table", - "input": "Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'", - "expected_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_readQuery_ambiguous_vs_tablePreview", - "case_type": "ambiguous_selection", - "description": "Prompt with a WHERE clause should use base_readQuery, not base_tablePreview", - "input": "Get me all orders from demo_user.evals_orders where the amount is greater than 500", - "expected_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_readQuery_missing_sql", - "case_type": "missing_parameter", - "description": "Agent must ask for clarification when no SQL or table is given", - "input": "Query the database for me", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_happy", - "case_type": "happy_path", - "description": "List tables in the evals database", - "input": "What tables are in the demo_user database?", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_missing_database", - "case_type": "missing_parameter", - "description": "User asks for tables without specifying a database", - "input": "Show me what tables are available", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableDDL_happy", - "case_type": "happy_path", - "description": "Retrieve DDL for the evals employees table", - "input": "Show me the CREATE statement for evals_employees in the demo_user database", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableDDL_ambiguous_vs_columnMetadata", - "case_type": "ambiguous_selection", - "description": "Asking for full structure including constraints should prefer base_tableDDL", - "input": "Describe the full structure of demo_user.evals_employees including any constraints and indexes", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnMetadata_happy", - "case_type": "happy_path", - "description": "Get precise Teradata type codes, character sets, and precision for columns in evals_orders", - "input": "Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnMetadata_ambiguous_vs_tableDDL", - "case_type": "ambiguous_selection", - "description": "Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL", - "input": "I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_happy", - "case_type": "happy_path", - "description": "Preview a sample of rows from the evals orders table", - "input": "Give me a quick preview of what's in demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_ambiguous_vs_readQuery", - "case_type": "ambiguous_selection", - "description": "A simple 'show me a few rows' without a WHERE clause should use base_tablePreview", - "input": "Show me a few rows from demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_databaseList_happy", - "case_type": "happy_path", - "description": "List all accessible databases", - "input": "What databases do I have access to?", - "expected_tools": [ - { - "name": "base_databaseList", - "params": {} - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_ambiguous_vs_databaseList", - "case_type": "ambiguous_selection", - "description": "Listing tables inside one database should prefer base_tableList, not base_databaseList", - "input": "What tables exist inside the demo_user database?", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_databaseList_ambiguous_vs_tableList", - "case_type": "ambiguous_selection", - "description": "Listing accessible databases should prefer base_databaseList, not base_tableList", - "input": "Which databases am I allowed to connect to on this system?", - "expected_tools": [ - { - "name": "base_databaseList", - "params": {} - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_saveDDL_ambiguous_vs_tableDDL", - "case_type": "ambiguous_selection", - "description": "Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL", - "input": "Export the DDL for demo_user.evals_orders to a file on disk", - "expected_tools": [ - { - "name": "base_saveDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableAffinity_ambiguous_vs_tableUsage", - "case_type": "ambiguous_selection", - "description": "Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage", - "input": "Which tables in demo_user tend to appear in the same queries as evals_employees?", - "expected_tools": [ - { - "name": "base_tableAffinity", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableUsage_ambiguous_vs_tableAffinity", - "case_type": "ambiguous_selection", - "description": "Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity", - "input": "Who has been reading tables in demo_user lately and how often?", - "expected_tools": [ - { - "name": "base_tableUsage", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_saveDDL_happy", - "case_type": "happy_path", - "description": "Save DDL for the evals employees table to a file", - "input": "Save the DDL for evals_employees in demo_user to a file", - "expected_tools": [ - { - "name": "base_saveDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableAffinity_happy", - "case_type": "happy_path", - "description": "Find tables commonly queried together with evals_orders", - "input": "Which tables in demo_user are most often queried together with evals_orders?", - "expected_tools": [ - { - "name": "base_tableAffinity", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableUsage_happy", - "case_type": "happy_path", - "description": "Check usage metrics for the evals database", - "input": "How frequently are the tables in demo_user being accessed and by whom?", - "expected_tools": [ - { - "name": "base_tableUsage", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_multi_tool_list_then_preview", - "case_type": "multi_tool", - "description": "List tables in evals database then preview one", - "input": "List all tables in demo_user, then show me a preview of evals_employees", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - }, - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_multi_tool_ddl_then_columns", - "case_type": "multi_tool", - "description": "Get DDL then drill into column detail for the same table", - "input": "First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - }, - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableDDL_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask for database name when only table is provided", - "input": "Show me the CREATE statement for evals_employees", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask which table to preview", - "input": "Preview some rows", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_saveDDL_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask which table to save DDL for", - "input": "Save the DDL to a file", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which table to preview, then calls base_tablePreview after user clarifies", - "input": "Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which database, then calls base_tableList after user clarifies", - "input": "Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_readQuery_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks for SQL, then calls base_readQuery after user provides the query", - "input": "Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnDescription_happy", - "case_type": "happy_path", - "description": "Show basic column info for the evals employees table", - "input": "Show me the columns in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnDescription_ambiguous_vs_columnMetadata", - "case_type": "ambiguous_selection", - "description": "A simple 'what columns does this table have' question should prefer base_columnDescription", - "input": "What are the column names and types in demo_user.evals_orders?", - "expected_tools": [ - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnMetadata_ambiguous_vs_columnDescription", - "case_type": "ambiguous_selection", - "description": "Asking for precise Teradata type codes and character sets should prefer base_columnMetadata", - "input": "Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_unknown_tool_missing", - "case_type": "missing_parameter", - "description": "User asks to do something no tool supports \u2014 agent should explain and ask for clarification", - "input": "Automatically tune my Teradata database indexes and rebalance the AMPs", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - } - ] -} diff --git a/tests/eval_test/results/latest_summary.md b/tests/eval_test/results/latest_summary.md deleted file mode 100644 index 9973a9d..0000000 --- a/tests/eval_test/results/latest_summary.md +++ /dev/null @@ -1,867 +0,0 @@ -# Teradata MCP Eval Run Summary - -**Started (UTC):** 2026-06-24T00:06:59.222908+00:00 -**Module filter:** base -**Case type filter:** all -**Agent model:** anthropic.claude-3-5-sonnet-20241022-v2:0 -**Judge model:** anthropic.claude-3-5-sonnet-20241022-v2:0 -**Eval database:** demo_user -**Tool descriptions:** live MCP server (baseline) - -## Overview - -| Metric | Count | -| --- | ---: | -| Total cases | 32 | -| Passed | 0 | -| Failed | 32 | - -## Failed cases - -### base_readQuery_happy (happy_path) - -**Description:** Execute a SQL query against the evals employees table - -**Eval prompt:** - -> Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales' - -**Expected tool(s):** - -- `base_readQuery` with params: -```json -{ - "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_readQuery_ambiguous_vs_tablePreview (ambiguous_selection) - -**Description:** Prompt with a WHERE clause should use base_readQuery, not base_tablePreview - -**Eval prompt:** - -> Get me all orders from demo_user.evals_orders where the amount is greater than 500 - -**Expected tool(s):** - -- `base_readQuery` with params: -```json -{ - "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_readQuery_missing_sql (missing_parameter) - -**Description:** Agent must ask for clarification when no SQL or table is given - -**Eval prompt:** - -> Query the database for me - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_happy (happy_path) - -**Description:** List tables in the evals database - -**Eval prompt:** - -> What tables are in the demo_user database? - -**Expected tool(s):** - -- `base_tableList` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_missing_database (missing_parameter) - -**Description:** User asks for tables without specifying a database - -**Eval prompt:** - -> Show me what tables are available - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableDDL_happy (happy_path) - -**Description:** Retrieve DDL for the evals employees table - -**Eval prompt:** - -> Show me the CREATE statement for evals_employees in the demo_user database - -**Expected tool(s):** - -- `base_tableDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableDDL_ambiguous_vs_columnMetadata (ambiguous_selection) - -**Description:** Asking for full structure including constraints should prefer base_tableDDL - -**Eval prompt:** - -> Describe the full structure of demo_user.evals_employees including any constraints and indexes - -**Expected tool(s):** - -- `base_tableDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnMetadata_happy (happy_path) - -**Description:** Get precise Teradata type codes, character sets, and precision for columns in evals_orders - -**Eval prompt:** - -> Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders - -**Expected tool(s):** - -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnMetadata_ambiguous_vs_tableDDL (ambiguous_selection) - -**Description:** Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL - -**Eval prompt:** - -> I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees - -**Expected tool(s):** - -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_happy (happy_path) - -**Description:** Preview a sample of rows from the evals orders table - -**Eval prompt:** - -> Give me a quick preview of what's in demo_user.evals_orders - -**Expected tool(s):** - -- `base_tablePreview` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_ambiguous_vs_readQuery (ambiguous_selection) - -**Description:** A simple 'show me a few rows' without a WHERE clause should use base_tablePreview - -**Eval prompt:** - -> Show me a few rows from demo_user.evals_orders - -**Expected tool(s):** - -- `base_tablePreview` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_databaseList_happy (happy_path) - -**Description:** List all accessible databases - -**Eval prompt:** - -> What databases do I have access to? - -**Expected tool(s):** - -- `base_databaseList` with params: -```json -{} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_ambiguous_vs_databaseList (ambiguous_selection) - -**Description:** Listing tables inside one database should prefer base_tableList, not base_databaseList - -**Eval prompt:** - -> What tables exist inside the demo_user database? - -**Expected tool(s):** - -- `base_tableList` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_databaseList_ambiguous_vs_tableList (ambiguous_selection) - -**Description:** Listing accessible databases should prefer base_databaseList, not base_tableList - -**Eval prompt:** - -> Which databases am I allowed to connect to on this system? - -**Expected tool(s):** - -- `base_databaseList` with params: -```json -{} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_saveDDL_ambiguous_vs_tableDDL (ambiguous_selection) - -**Description:** Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL - -**Eval prompt:** - -> Export the DDL for demo_user.evals_orders to a file on disk - -**Expected tool(s):** - -- `base_saveDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableAffinity_ambiguous_vs_tableUsage (ambiguous_selection) - -**Description:** Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage - -**Eval prompt:** - -> Which tables in demo_user tend to appear in the same queries as evals_employees? - -**Expected tool(s):** - -- `base_tableAffinity` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableUsage_ambiguous_vs_tableAffinity (ambiguous_selection) - -**Description:** Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity - -**Eval prompt:** - -> Who has been reading tables in demo_user lately and how often? - -**Expected tool(s):** - -- `base_tableUsage` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_saveDDL_happy (happy_path) - -**Description:** Save DDL for the evals employees table to a file - -**Eval prompt:** - -> Save the DDL for evals_employees in demo_user to a file - -**Expected tool(s):** - -- `base_saveDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableAffinity_happy (happy_path) - -**Description:** Find tables commonly queried together with evals_orders - -**Eval prompt:** - -> Which tables in demo_user are most often queried together with evals_orders? - -**Expected tool(s):** - -- `base_tableAffinity` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableUsage_happy (happy_path) - -**Description:** Check usage metrics for the evals database - -**Eval prompt:** - -> How frequently are the tables in demo_user being accessed and by whom? - -**Expected tool(s):** - -- `base_tableUsage` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_multi_tool_list_then_preview (multi_tool) - -**Description:** List tables in evals database then preview one - -**Eval prompt:** - -> List all tables in demo_user, then show me a preview of evals_employees - -**Expected tool(s):** - -- `base_tableList` with params: -```json -{ - "database_name": "demo_user" -} -``` -- `base_tablePreview` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_multi_tool_ddl_then_columns (multi_tool) - -**Description:** Get DDL then drill into column detail for the same table - -**Eval prompt:** - -> First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata - -**Expected tool(s):** - -- `base_tableDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableDDL_missing_table (missing_parameter) - -**Description:** Agent must ask for database name when only table is provided - -**Eval prompt:** - -> Show me the CREATE statement for evals_employees - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_missing_table (missing_parameter) - -**Description:** Agent must ask which table to preview - -**Eval prompt:** - -> Preview some rows - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_saveDDL_missing_table (missing_parameter) - -**Description:** Agent must ask which table to save DDL for - -**Eval prompt:** - -> Save the DDL to a file - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_clarify_then_call (missing_parameter) - -**Description:** Agent asks which table to preview, then calls base_tablePreview after user clarifies - -**Eval prompt:** - -> Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_clarify_then_call (missing_parameter) - -**Description:** Agent asks which database, then calls base_tableList after user clarifies - -**Eval prompt:** - -> Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_readQuery_clarify_then_call (missing_parameter) - -**Description:** Agent asks for SQL, then calls base_readQuery after user provides the query - -**Eval prompt:** - -> Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales' - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnDescription_happy (happy_path) - -**Description:** Show basic column info for the evals employees table - -**Eval prompt:** - -> Show me the columns in demo_user.evals_employees - -**Expected tool(s):** - -- `base_columnDescription` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnDescription_ambiguous_vs_columnMetadata (ambiguous_selection) - -**Description:** A simple 'what columns does this table have' question should prefer base_columnDescription - -**Eval prompt:** - -> What are the column names and types in demo_user.evals_orders? - -**Expected tool(s):** - -- `base_columnDescription` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnMetadata_ambiguous_vs_columnDescription (ambiguous_selection) - -**Description:** Asking for precise Teradata type codes and character sets should prefer base_columnMetadata - -**Eval prompt:** - -> Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees - -**Expected tool(s):** - -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_unknown_tool_missing (missing_parameter) - -**Description:** User asks to do something no tool supports — agent should explain and ask for clarification - -**Eval prompt:** - -> Automatically tune my Teradata database indexes and rebalance the AMPs - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -## Passed cases - -_None._ - diff --git a/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/manifest.json b/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/manifest.json deleted file mode 100644 index 6bed444..0000000 --- a/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/manifest.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "run_id": "2026-06-23T21-14-19.042368Z__all__baseline", - "started_at": "2026-06-23T21:14:19.042368+00:00", - "module_filter": "all", - "case_type_filter": "all", - "run_label": null, - "description_mode": "mcp_server", - "description_overrides_file": null, - "description_override_count": 0, - "agent_model_id": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", - "judge_model_id": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", - "evals_database": "demo_user", - "total": 106, - "passed": 103, - "failed": 3, - "artifacts": { - "summary_md": "summary.md", - "summary_json": "summary.json", - "manifest": "manifest.json", - "suggested_overrides": "suggested_overrides.json" - } -} diff --git a/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.json b/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.json deleted file mode 100644 index 6b18f98..0000000 --- a/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.json +++ /dev/null @@ -1,3209 +0,0 @@ -{ - "run_id": "2026-06-23T21-14-19.042368Z__all__baseline", - "started_at": "2026-06-23T21:14:19.042368+00:00", - "module_filter": "all", - "case_type_filter": "all", - "run_label": null, - "agent_model_id": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", - "judge_model_id": "us.anthropic.claude-sonnet-4-5-20250929-v1:0", - "evals_database": "demo_user", - "description_mode": "mcp_server", - "description_overrides_file": null, - "description_override_count": 0, - "total": 106, - "passed": 103, - "failed": 3, - "cases": [ - { - "case_id": "base_readQuery_happy", - "case_type": "happy_path", - "description": "Execute a SQL query against the evals employees table", - "input": "Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'", - "expected_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" - } - } - ], - "actual_output": "Here are the results from the query:\n\n| Name | Department | Salary |\n|------|-----------|--------|\n| Bob Smith | Sales | $72,000 |\n| Frank Lee | Sales | $69,500 |\n\nThe query returned **2 employees** in the Sales department. Bob Smith has the highest salary at $72,000, followed by Frank Lee at $69,500.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_readQuery_ambiguous_vs_tablePreview", - "case_type": "ambiguous_selection", - "description": "Prompt with a WHERE clause should use base_readQuery, not base_tablePreview", - "input": "Get me all orders from demo_user.evals_orders where the amount is greater than 500", - "expected_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" - } - } - ], - "actual_output": "I found **8 orders** from demo_user.evals_orders where the amount is greater than 500:\n\n| Order ID | Customer Name | Order Date | Ship Date | Amount | Product Category | Quantity |\n|----------|--------------|------------|-----------|---------|-----------------|----------|\n| 1009 | Iota Inc | 2024-05-08 | - | $1,780.00 | Electronics | 1 |\n| 1005 | Epsilon LLC | 2024-03-01 | 2024-03-04 | $560.00 | Books | 8 |\n| 1010 | Kappa Ltd | 2024-05-25 | 2024-05-28 | $4,200.00 | Sports | 7 |\n| 1007 | Eta Group | 2024-04-02 | 2024-04-06 | $2,100.00 | Sports | 4 |\n| 1001 | Acme Corp | 2024-01-05 | 2024-01-08 | $1,250.00 | Electronics | 3 |\n| 1008 | Theta Corp | 2024-04-20 | 2024-04-23 | $930.00 | Food | 6 |\n| 1004 | Delta Co | 2024-02-17 | 2024-02-20 | $3,400.00 | Electronics | 2 |\n| 1003 | Gamma Inc | 2024-02-03 | - | $875.50 | Food | 5 |\n\nThe highest value order is from **Kappa Ltd** at $4,200.00, and the orders range from $560.00 to $4,200.00.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_readQuery_missing_sql", - "case_type": "missing_parameter", - "description": "Agent must ask for clarification when no SQL or table is given", - "input": "Query the database for me", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you query the database! However, I need a bit more information to proceed:\n\n1. **What would you like to query?** For example:\n - A specific table (e.g., \"show me data from the sales table\")\n - List available databases or tables\n - Get information about table structure/columns\n - Run a specific SQL query you have in mind\n - View sample data from a table\n - Get statistics or summaries\n\n2. **Which database and/or table** are you interested in? (if you know the names)\n\nCould you provide more details about what you'd like to see or accomplish?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tableList_happy", - "case_type": "happy_path", - "description": "List tables in the evals database", - "input": "What tables are in the demo_user database?", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - } - ], - "actual_output": "The **demo_user** database contains **2 tables**:\n\n1. **evals_orders**\n2. **evals_employees**\n\nWould you like to see more details about either of these tables, such as their columns, sample data, or structure?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tableList_missing_database", - "case_type": "missing_parameter", - "description": "User asks for tables without specifying a database", - "input": "Show me what tables are available", - "expected_tools": [], - "passed": false, - "failure_stage": "deterministic", - "failure_detail": "expected no tool calls for missing_parameter case, got ['base_tableList']", - "actual_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "" - } - } - ], - "actual_output": "I can see the system has many tables available. The results show tables across multiple databases. Here's a summary of what I found:\n\n**Main Databases with Tables:**\n- **DBC** - System database with metadata tables (InDoubtLog, StatsTbl, QryLog, DBCInfo, etc.)\n- **TD_SYSFNLIB** - System function library\n- **SYSLIB** - System library\n- **TD_SERVER_DB** - Server database\n- **SQLJ** - SQL/Java support\n- **TD_SYSGPL** - System GPL (General Purpose Library)\n- **SystemFe** - System functions\n- **SYSSPATIAL** - Spatial database functions\n- **SYSUDTLIB** - User-defined types library\n- **SYSBAR** - System backup and recovery\n- **TD_SYSXML** - XML functions\n- **TDMaps** - Map functions\n- **Crashdumps** - System crash dump storage\n\nThe list shows **hundreds of system tables and views** primarily from the DBC (Data Base Control) system database, which contains metadata about databases, tables, users, permissions, query logs, and system configuration.\n\nWould you like me to:\n1. Show tables from a specific database?\n2. Filter to show only user databases (excluding system databases)?\n3. Show details about a specific table?", - "metric_reasons": [], - "recommendation": "The prompt deliberately omits required information: \"Show me what tables are available\". The agent called `base_tableList` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response.", - "turn_details": null - }, - { - "case_id": "base_tableDDL_happy", - "case_type": "happy_path", - "description": "Retrieve DDL for the evals employees table", - "input": "Show me the CREATE statement for evals_employees in the demo_user database", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tableDDL", - "params": { - "table_name": "evals_employees", - "database_name": "demo_user" - } - } - ], - "actual_output": "Here's the CREATE statement for the `evals_employees` table in the `demo_user` database:\n\n```sql\nCREATE SET TABLE demo_user.evals_employees ,FALLBACK ,\n NO BEFORE JOURNAL,\n NO AFTER JOURNAL,\n CHECKSUM = DEFAULT,\n DEFAULT MERGEBLOCKRATIO,\n MAP = TD_MAP1\n (\n employee_id INTEGER NOT NULL,\n name VARCHAR(100) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n department VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n salary DECIMAL(10,2),\n region VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n hire_date DATE FORMAT 'YY/MM/DD' NOT NULL,\n manager_id INTEGER, \nPRIMARY KEY ( employee_id ));\n```\n\nThis table contains employee information with:\n- **employee_id** (INTEGER) - Primary key\n- **name** (VARCHAR 100) - Employee name\n- **department** (VARCHAR 50) - Department name\n- **salary** (DECIMAL 10,2) - Salary amount\n- **region** (VARCHAR 50) - Geographic region\n- **hire_date** (DATE) - Date of hire\n- **manager_id** (INTEGER) - Reference to manager\n\nThe table is configured with FALLBACK for data protection and has no journal logging enabled.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tableDDL_ambiguous_vs_columnMetadata", - "case_type": "ambiguous_selection", - "description": "Asking for full structure including constraints should prefer base_tableDDL", - "input": "Describe the full structure of demo_user.evals_employees including any constraints and indexes", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tableDDL", - "params": { - "table_name": "evals_employees", - "database_name": "demo_user" - } - } - ], - "actual_output": "## Full Structure of `demo_user.evals_employees`\n\nHere's the complete DDL for the table:\n\n```sql\nCREATE SET TABLE demo_user.evals_employees ,FALLBACK ,\n NO BEFORE JOURNAL,\n NO AFTER JOURNAL,\n CHECKSUM = DEFAULT,\n DEFAULT MERGEBLOCKRATIO,\n MAP = TD_MAP1\n (\n employee_id INTEGER NOT NULL,\n name VARCHAR(100) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n department VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n salary DECIMAL(10,2),\n region VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n hire_date DATE FORMAT 'YY/MM/DD' NOT NULL,\n manager_id INTEGER, \nPRIMARY KEY ( employee_id ))\n;\n```\n\n### **Columns:**\n\n| Column | Data Type | Constraints |\n|--------|-----------|-------------|\n| **employee_id** | INTEGER | NOT NULL, PRIMARY KEY |\n| **name** | VARCHAR(100) LATIN | NOT NULL, NOT CASESPECIFIC |\n| **department** | VARCHAR(50) LATIN | NOT NULL, NOT CASESPECIFIC |\n| **salary** | DECIMAL(10,2) | Nullable |\n| **region** | VARCHAR(50) LATIN | NOT NULL, NOT CASESPECIFIC |\n| **hire_date** | DATE | NOT NULL, Format: 'YY/MM/DD' |\n| **manager_id** | INTEGER | Nullable |\n\n### **Constraints:**\n- **PRIMARY KEY**: `employee_id` - Ensures unique employee identification\n\n### **Indexes:**\n- The PRIMARY KEY on `employee_id` implicitly creates a Unique Primary Index (UPI)\n\n### **Table Attributes:**\n- **SET TABLE**: Duplicate rows are not allowed\n- **FALLBACK**: Data is protected with fallback copies\n- **NO BEFORE JOURNAL**: No before-image journaling\n- **NO AFTER JOURNAL**: No after-image journaling\n- **CHECKSUM = DEFAULT**: Uses default checksum validation\n- **DEFAULT MERGEBLOCKRATIO**: Uses system default for block merging\n- **MAP = TD_MAP1**: Uses TD_MAP1 for data distribution\n\nThis table appears to be designed for employee management with hierarchical relationships (via `manager_id`) and includes demographic and compensation information.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_columnMetadata_happy", - "case_type": "happy_path", - "description": "Get precise Teradata type codes, character sets, and precision for columns in evals_orders", - "input": "Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_orders" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_orders" - } - } - ], - "actual_output": "Here are the exact Teradata type codes, character sets, and precision details for every column in **demo_user.evals_orders**:\n\n| Column Name | Type Code | Column Type | Character Set | Length | Decimal Precision | Nullable | Index Type |\n|-------------|-----------|-------------|---------------|---------|-------------------|----------|------------|\n| **order_id** | I | INTEGER | - | 4 bytes | - | N | USI (Unique Secondary Index) |\n| **customer_name** | CV | VARCHAR(100) LATIN | LATIN | 100 | - | N | - |\n| **order_date** | DA | DATE | - | 4 bytes | - | N | - |\n| **ship_date** | DA | DATE | - | 4 bytes | - | Y | - |\n| **amount** | D | DOUBLE PRECISION | - | 8 bytes | 10 total digits, 2 fractional | N | - |\n| **product_category** | CV | VARCHAR(50) LATIN | LATIN | 50 | - | N | - |\n| **quantity** | I | INTEGER | - | 4 bytes | - | N | - |\n\n**Key Details:**\n- **Type Codes**: I = Integer, CV = Variable Character, DA = Date, D = Double Precision (Float)\n- **Character Sets**: Both VARCHAR columns use LATIN character set (CharType = 1)\n- **Decimal Precision**: The `amount` column (type D) has 10 total digits with 2 fractional digits\n- **Indexes**: `order_id` has a Unique Secondary Index (USI)\n- **Nullability**: Only `ship_date` allows NULL values; all other columns are NOT NULL", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_columnMetadata_ambiguous_vs_tableDDL", - "case_type": "ambiguous_selection", - "description": "Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL", - "input": "I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_employees" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_employees", - "fields": "ColumnName,ColumnType,Nullable,CharType" - } - } - ], - "actual_output": "Perfect! Here's the character set and nullability information for every column in **demo_user.evals_employees**:\n\n| Column Name | Data Type | Character Set | Nullable |\n|------------|-----------|---------------|----------|\n| **employee_id** | INTEGER | N/A (numeric) | **No** (NOT NULL) |\n| **name** | VARCHAR(100) | **LATIN** | **No** (NOT NULL) |\n| **department** | VARCHAR(50) | **LATIN** | **No** (NOT NULL) |\n| **salary** | DOUBLE PRECISION | N/A (numeric) | **Yes** (NULLABLE) |\n| **region** | VARCHAR(50) | **LATIN** | **No** (NOT NULL) |\n| **hire_date** | DATE | N/A (date) | **No** (NOT NULL) |\n| **manager_id** | INTEGER | N/A (numeric) | **Yes** (NULLABLE) |\n\n**Summary:**\n- **All character columns use LATIN character set** (no UNICODE columns found)\n- **Character columns:** name, department, region (all LATIN)\n- **Nullable columns:** salary, manager_id (2 columns)\n- **NOT NULL columns:** employee_id, name, department, region, hire_date (5 columns)", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tablePreview_happy", - "case_type": "happy_path", - "description": "Preview a sample of rows from the evals orders table", - "input": "Give me a quick preview of what's in demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tablePreview", - "params": { - "table_name": "evals_orders", - "database_name": "demo_user" - } - } - ], - "actual_output": "Here's a quick preview of the **demo_user.evals_orders** table:\n\n| order_id | customer_name | order_date | ship_date | amount | product_category | quantity |\n|----------|---------------|------------|-----------|---------|------------------|----------|\n| 1009 | Iota Inc | 2024-05-08 | null | 1780.0 | Electronics | 1 |\n| 1002 | Beta Ltd | 2024-01-12 | 2024-01-15 | -200.0 | Clothing | 1 |\n| 1010 | Kappa Ltd | 2024-05-25 | 2024-05-28 | 4200.0 | Sports | 7 |\n| 1008 | Theta Corp | 2024-04-20 | 2024-04-23 | 930.0 | Food | 6 |\n| 1007 | Eta Group | 2024-04-02 | 2024-04-06 | 2100.0 | Sports | 4 |\n\n**Table Structure:**\n- **order_id** (int) - Order identifier\n- **customer_name** (str) - Customer name\n- **order_date** (date) - Date order was placed\n- **ship_date** (date) - Date order was shipped (some null values)\n- **amount** (Decimal) - Order amount (note: some negative values present)\n- **product_category** (str) - Product category (Electronics, Clothing, Sports, Food, etc.)\n- **quantity** (int) - Order quantity\n\nThe table contains order transaction data with categories like Electronics, Clothing, Sports, and Food. I notice there are some null ship dates and at least one negative amount value (-200.0).", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tablePreview_ambiguous_vs_readQuery", - "case_type": "ambiguous_selection", - "description": "A simple 'show me a few rows' without a WHERE clause should use base_tablePreview", - "input": "Show me a few rows from demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tablePreview", - "params": { - "table_name": "evals_orders", - "database_name": "demo_user" - } - } - ], - "actual_output": "Here are a few sample rows from the **demo_user.evals_orders** table:\n\n| order_id | customer_name | order_date | ship_date | amount | product_category | quantity |\n|----------|---------------|------------|------------|----------|------------------|----------|\n| 1002 | Beta Ltd | 2024-01-12 | 2024-01-15 | -200.0 | Clothing | 1 |\n| 1005 | Epsilon LLC | 2024-01-12 | 2024-03-04 | 560.0 | Books | 8 |\n| 1010 | Kappa Ltd | 2024-05-25 | 2024-05-28 | 4200.0 | Sports | 7 |\n| 1008 | Theta Corp | 2024-04-20 | 2024-04-23 | 930.0 | Food | 6 |\n| 1006 | Zeta Partners | 2024-03-15 | *null* | -150.0 | Clothing | 1 |\n\nThe table contains order information with columns for order ID, customer name, order and ship dates, amount, product category, and quantity. I notice there are some negative amounts and one missing ship_date value in this sample.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_databaseList_happy", - "case_type": "happy_path", - "description": "List all accessible databases", - "input": "What databases do I have access to?", - "expected_tools": [ - { - "name": "base_databaseList", - "params": {} - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_databaseList", - "params": { - "scope": "user" - } - } - ], - "actual_output": "You have access to **10 databases** on this Teradata system:\n\n| Database Name | Type | Description |\n|--------------|------|-------------|\n| **TD_SYSXML** | DataBase | - |\n| **gs_tables_db** | DataBase | - |\n| **mldb** | User | - |\n| **HEALTHCARE_DEMO** | DataBase | - |\n| **modelops** | DataBase | - |\n| **td_tapidb** | User | - |\n| **GLOBAL_FUNCTIONS** | DataBase | - |\n| **DELTA_DEMO** | DataBase | - |\n| **demo_user** | User | Logon ID for the end user to run demonstrations |\n| **TD_SYSAI** | DataBase | - |\n\nThe databases include:\n- **User databases**: `mldb`, `td_tapidb`, and `demo_user` (user accounts with database space)\n- **Application databases**: `HEALTHCARE_DEMO`, `DELTA_DEMO`, `modelops`, `gs_tables_db`\n- **System/function databases**: `TD_SYSXML`, `TD_SYSAI`, `GLOBAL_FUNCTIONS`\n\nWould you like to explore the tables in any of these databases?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tableList_ambiguous_vs_databaseList", - "case_type": "ambiguous_selection", - "description": "Listing tables inside one database should prefer base_tableList, not base_databaseList", - "input": "What tables exist inside the demo_user database?", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - } - ], - "actual_output": "The **demo_user** database contains **2 tables**:\n\n1. **evals_orders**\n2. **evals_employees**\n\nWould you like to see more details about either of these tables, such as their column definitions or sample data?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_databaseList_ambiguous_vs_tableList", - "case_type": "ambiguous_selection", - "description": "Listing accessible databases should prefer base_databaseList, not base_tableList", - "input": "Which databases am I allowed to connect to on this system?", - "expected_tools": [ - { - "name": "base_databaseList", - "params": {} - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_databaseList", - "params": { - "scope": "user" - } - } - ], - "actual_output": "Based on the query results, you have access to **10 databases** on this Teradata system:\n\n## User Databases (User type):\n1. **mldb** - Machine Learning Database\n2. **td_tapidb** - Teradata API Database \n3. **demo_user** - Logon ID for end users to run demonstrations\n\n## Standard Databases (Database type):\n4. **TD_SYSXML** - Teradata System XML database\n5. **gs_tables_db** - Golden Standard tables database\n6. **HEALTHCARE_DEMO** - Healthcare demonstration database\n7. **modelops** - Model Operations database\n8. **GLOBAL_FUNCTIONS** - Global functions database\n9. **DELTA_DEMO** - Delta demonstration database\n10. **TD_SYSAI** - Teradata System AI database\n\nThe databases include a mix of system databases (TD_SYSXML, TD_SYSAI, GLOBAL_FUNCTIONS), machine learning and analytics databases (mldb, modelops), and demonstration databases (HEALTHCARE_DEMO, DELTA_DEMO, demo_user). You can connect to any of these databases based on your access permissions.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_saveDDL_ambiguous_vs_tableDDL", - "case_type": "ambiguous_selection", - "description": "Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL", - "input": "Export the DDL for demo_user.evals_orders to a file on disk", - "expected_tools": [ - { - "name": "base_saveDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_saveDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders", - "object_type": "TABLE" - } - } - ], - "actual_output": "Perfect! The DDL for `demo_user.evals_orders` has been successfully exported to disk.\n\n**File Details:**\n- **Location**: `/Users/Daniel.Tehan/Code/MCP/teradata-mcp-server/ddls_extracted/evals_orders_DDL.sql`\n- **File size**: 721 bytes\n- **DDL size**: 563 characters\n- **Object type**: TABLE\n\nThe DDL file has been saved and is ready to use.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tableAffinity_ambiguous_vs_tableUsage", - "case_type": "ambiguous_selection", - "description": "Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage", - "input": "Which tables in demo_user tend to appear in the same queries as evals_employees?", - "expected_tools": [ - { - "name": "base_tableAffinity", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tableAffinity", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "actual_output": "Based on the analysis of query logs, **no tables have been found that appear in the same queries as `evals_employees`** in the `demo_user` database.\n\nThis could mean:\n- The `evals_employees` table hasn't been queried recently alongside other tables\n- Query logging may not have captured JOIN operations with this table\n- The table might be queried in isolation without JOINs to other tables\n- There may be insufficient query history in the system logs (DBC.DBQLObjTbl)\n\nIf you'd like, I can:\n1. Check if the `evals_employees` table exists and preview its structure\n2. Look at overall table usage in the `demo_user` database\n3. Examine what queries have been run against `evals_employees` specifically\n\nWould any of these help?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tableUsage_ambiguous_vs_tableAffinity", - "case_type": "ambiguous_selection", - "description": "Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity", - "input": "Who has been reading tables in demo_user lately and how often?", - "expected_tools": [ - { - "name": "base_tableUsage", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tableUsage", - "params": { - "database_name": "demo_user" - } - } - ], - "actual_output": "Based on the table usage data for the **demo_user** database, here's what I found:\n\n## Recent Table Access Activity\n\nTwo tables in demo_user have been actively queried recently:\n\n### 1. **EVALS_ORDERS** (Most Popular)\n- **Query Count**: 96 queries\n- **Usage Share**: 63% of all queries\n- **Usage Frequency**: High\n- **Activity Window**: First queried 4 days ago, last queried today (0 days ago)\n\n### 2. **EVALS_EMPLOYEES**\n- **Query Count**: 55 queries\n- **Usage Share**: 36% of all queries\n- **Usage Frequency**: High\n- **Activity Window**: First queried 4 days ago, last queried today (0 days ago)\n\n## Summary\nBoth tables show **high usage frequency** with consistent access over the past 4 days, including activity today. The EVALS_ORDERS table is being accessed almost twice as frequently as EVALS_EMPLOYEES.\n\nWould you like to see more details, such as:\n- Which specific users have been running these queries?\n- What types of SQL statements are being executed?\n- Access patterns by time of day?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_saveDDL_happy", - "case_type": "happy_path", - "description": "Save DDL for the evals employees table to a file", - "input": "Save the DDL for evals_employees in demo_user to a file", - "expected_tools": [ - { - "name": "base_saveDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_saveDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees", - "object_type": "TABLE" - } - } - ], - "actual_output": "Perfect! The DDL for the `evals_employees` table has been successfully saved to a file.\n\n**File Details:**\n- **Location**: `/Users/Daniel.Tehan/Code/MCP/teradata-mcp-server/ddls_extracted/evals_employees_DDL.sql`\n- **File size**: 736 bytes\n- **DDL content size**: 572 characters\n\nThe DDL file contains the complete CREATE TABLE statement for the `evals_employees` table from the `demo_user` database.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tableAffinity_happy", - "case_type": "happy_path", - "description": "Find tables commonly queried together with evals_orders", - "input": "Which tables in demo_user are most often queried together with evals_orders?", - "expected_tools": [ - { - "name": "base_tableAffinity", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tableAffinity", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "actual_output": "The query found no tables that are commonly queried together with `evals_orders` in the `demo_user` database. This could mean one of the following:\n\n1. **The table `evals_orders` hasn't been queried recently** - or at least not in queries that also referenced other tables\n2. **Query logging may not capture this table's usage** - the DBQLObjTbl system table only contains data if query logging is enabled and has captured queries involving this table\n3. **The table is queried in isolation** - queries may be selecting from `evals_orders` alone without JOINing to other tables\n\nTo investigate further, I can:\n- Check if the table exists and get its structure\n- Look at overall table usage patterns in the demo_user database\n- Check recent SQL queries that have accessed this table\n\nWould you like me to investigate any of these options?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tableUsage_happy", - "case_type": "happy_path", - "description": "Check usage metrics for the evals database", - "input": "How frequently are the tables in demo_user being accessed and by whom?", - "expected_tools": [ - { - "name": "base_tableUsage", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tableUsage", - "params": { - "database_name": "demo_user" - } - } - ], - "actual_output": "Based on the access patterns for tables in the **demo_user** database, here's what I found:\n\n## Table Access Frequency Summary\n\n| Table Name | Query Count | % of Total | Usage Frequency | First Query | Last Query |\n|------------|------------|------------|-----------------|-------------|------------|\n| **EVALS_ORDERS** | 96 | 63% | High | 4 days ago | Today |\n| **EVALS_EMPLOYEES** | 55 | 36% | High | 4 days ago | Today |\n\n### Key Insights:\n\n1. **EVALS_ORDERS** is the most heavily accessed table, accounting for **63% of all queries** against the demo_user database with 96 total queries.\n\n2. **EVALS_EMPLOYEES** is also frequently accessed, representing **36% of queries** with 55 total queries.\n\n3. Both tables show **High usage frequency** and have been actively queried within the last 4 days, with the most recent access being today.\n\n4. These two tables represent **100% of the tracked query activity** in the demo_user database during the monitoring period.\n\nThe data shows consistent, ongoing usage of both tables, suggesting they are core tables for active business operations or analysis. Would you like me to drill down further to see which specific users are accessing these tables, or analyze the types of queries being run against them?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_multi_tool_list_then_preview", - "case_type": "multi_tool", - "description": "List tables in evals database then preview one", - "input": "List all tables in demo_user, then show me a preview of evals_employees", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - }, - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - }, - { - "name": "base_tablePreview", - "params": { - "table_name": "evals_employees", - "database_name": "demo_user" - } - } - ], - "actual_output": "## Tables in demo_user Database\n\nThe **demo_user** database contains **2 tables**:\n\n1. **evals_orders**\n2. **evals_employees**\n\n---\n\n## Preview of evals_employees Table\n\nHere's a sample of 5 rows from the **evals_employees** table:\n\n| employee_id | name | department | salary | region | hire_date | manager_id |\n|-------------|---------------|-------------|-----------|--------|------------|------------|\n| 10 | Jack Wilson | Engineering | 88,000.00 | South | 2016-08-22 | 5 |\n| 4 | David Brown | Finance | 85,000.00 | West | 2021-01-10 | 1 |\n| 5 | Eva Martinez | Engineering | 102,000.00| North | 2017-06-05 | 1 |\n| 3 | Carol White | HR | 68,000.00 | East | 2018-11-20 | 1 |\n| 7 | Grace Kim | Marketing | 77,000.00 | East | 2020-09-30 | 1 |\n\nThe table contains employee information with the following columns:\n- **employee_id**: Employee identifier\n- **name**: Employee name\n- **department**: Department (Engineering, Finance, HR, Marketing)\n- **salary**: Employee salary\n- **region**: Geographic region (North, South, East, West)\n- **hire_date**: Date when the employee was hired\n- **manager_id**: ID of the employee's manager", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_multi_tool_ddl_then_columns", - "case_type": "multi_tool", - "description": "Get DDL then drill into column detail for the same table", - "input": "First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - }, - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_orders" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tableDDL", - "params": { - "table_name": "evals_orders", - "database_name": "demo_user" - } - }, - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_orders" - } - } - ], - "actual_output": "## DDL for demo_user.evals_orders\n\n```sql\nCREATE SET TABLE demo_user.evals_orders ,FALLBACK ,\n NO BEFORE JOURNAL,\n NO AFTER JOURNAL,\n CHECKSUM = DEFAULT,\n DEFAULT MERGEBLOCKRATIO,\n MAP = TD_MAP1\n (\n order_id INTEGER NOT NULL,\n customer_name VARCHAR(100) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n order_date DATE FORMAT 'YY/MM/DD' NOT NULL,\n ship_date DATE FORMAT 'YY/MM/DD',\n amount DECIMAL(10,2) NOT NULL,\n product_category VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,\n quantity INTEGER NOT NULL, \nPRIMARY KEY ( order_id ))\n;\n```\n\n## Detailed Column Metadata\n\n| Column Name | Type | Length | Nullable | Character Set | Decimal (Total, Fractional) | Format | Index Type | Case Specificity | Default Value |\n|-------------|------|--------|----------|---------------|----------------------------|--------|------------|------------------|---------------|\n| **order_id** | INTEGER | 4 | No | - | - | -(10)9 | **USI** | NOT CASESPECIFIC | null |\n| **customer_name** | VARCHAR(100) LATIN | 100 | No | LATIN | - | X(100) | - | NOT CASESPECIFIC | null |\n| **order_date** | DATE | 4 | No | - | - | YY/MM/DD | - | NOT CASESPECIFIC | null |\n| **ship_date** | DATE | 4 | **Yes** | - | - | YY/MM/DD | - | NOT CASESPECIFIC | null |\n| **amount** | DOUBLE PRECISION | 8 | No | - | (10, 2) | ---------.99 | - | NOT CASESPECIFIC | null |\n| **product_category** | VARCHAR(50) LATIN | 50 | No | LATIN | - | X(50) | - | NOT CASESPECIFIC | null |\n| **quantity** | INTEGER | 4 | No | - | - | -(10)9 | - | NOT CASESPECIFIC | null |\n\n### Key Findings:\n- **Primary Key**: `order_id` with a Unique Secondary Index (USI)\n- **Total Columns**: 7\n- **Nullable Columns**: Only `ship_date` allows NULL values\n- **Character Columns**: `customer_name` and `product_category` use LATIN character set\n- **Numeric Columns**: `order_id`, `quantity` (INTEGER), and `amount` (DECIMAL with 10 total digits, 2 fractional)\n- **Date Columns**: `order_date` and `ship_date` (both formatted as YY/MM/DD)", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tableDDL_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask for database name when only table is provided", - "input": "Show me the CREATE statement for evals_employees", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I need both the table name and database name to retrieve the CREATE statement. You've provided the table name \"evals_employees\", but which database does this table belong to?\n\nCould you please provide the database name?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tablePreview_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask which table to preview", - "input": "Preview some rows", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you preview some rows from a table! However, I need to know which table you'd like to see.\n\nCould you please provide:\n1. **Table name** - The name of the table you want to preview\n2. **Database name** (optional) - The database where the table is located\n\nFor example: \"Preview rows from customer_data in the sales database\" or just \"Preview the orders table\"", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_saveDDL_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask which table to save DDL for", - "input": "Save the DDL to a file", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you save DDL to a file! However, I need some information first:\n\n1. **Database name** - Which database contains the object?\n2. **Object name** - Which table, view, or stored procedure do you want to extract DDL for?\n3. **Object type** - Is it a TABLE, VIEW, or PROCEDURE? (default is PROCEDURE)\n4. **Output directory** (optional) - Where should the file be saved? (default is './ddls_extracted')\n\nPlease provide at least the database name and object name so I can save the DDL for you.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_tablePreview_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which table to preview, then calls base_tablePreview after user clarifies", - "input": "Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_tablePreview", - "params": { - "table_name": "evals_employees", - "database_name": "demo_user" - } - } - ], - "actual_output": "Here's a preview of the **demo_user.evals_employees** table:\n\n| employee_id | name | department | salary | region | hire_date | manager_id |\n|-------------|----------------|--------------|----------|--------|------------|------------|\n| 9 | Isla Thompson | HR | *null* | North | 2023-02-14 | 3 |\n| 7 | Grace Kim | Marketing | 77000.0 | East | 2020-09-30 | 1 |\n| 4 | David Brown | Finance | 85000.0 | West | 2021-01-10 | 1 |\n| 6 | Frank Lee | Sales | 69500.0 | South | 2022-04-18 | 2 |\n| 10 | Jack Wilson | Engineering | 88000.0 | South | 2016-08-22 | 5 |\n\n**Table Overview:**\n- **7 columns**: employee_id, name, department, salary, region, hire_date, manager_id\n- Contains employee information including their department, salary, region, and hire dates\n- Note: Isla Thompson (employee_id 9) has a null salary value\n- The table appears to track employees across different departments (HR, Marketing, Finance, Sales, Engineering) and regions\n\nWould you like to see more rows or perform any analysis on this data?", - "metric_reasons": [], - "recommendation": null, - "turn_details": [ - { - "turn": 1, - "input": "Preview some rows for me", - "mode": "clarification", - "passed": true, - "actual_tools": [] - }, - { - "turn": 2, - "input": "Preview rows from demo_user.evals_employees", - "mode": "tool", - "passed": true, - "actual_tools": [ - { - "name": "base_tablePreview", - "params": { - "table_name": "evals_employees", - "database_name": "demo_user" - } - } - ] - } - ] - }, - { - "case_id": "base_tableList_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which database, then calls base_tableList after user clarifies", - "input": "Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database", - "expected_tools": [], - "passed": false, - "failure_stage": "deterministic", - "failure_detail": "turn 1: expected no tool calls for missing_parameter case, got ['base_tableList']", - "actual_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "" - } - } - ], - "actual_output": "I can see there are tables available in the system. The results show many tables from the **DBC** database (which is the Teradata Data Dictionary/system catalog). \n\nWould you like me to:\n1. **Filter by a specific database** - If you're looking for user/application tables rather than system tables, please tell me which database to show\n2. **Show only user databases** - I can list the available databases first so you can pick one\n3. **Continue showing all tables** - though the full list is quite long\n\nMost users are typically interested in their own application databases rather than the DBC system tables. Which would you prefer?", - "metric_reasons": [], - "recommendation": "The prompt deliberately omits required information: \"(multi-turn \u2014 see turn details)\". The agent called `base_tableList` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response.", - "turn_details": [ - { - "turn": 1, - "input": "Show me what tables are available", - "mode": "clarification", - "passed": false, - "failure_stage": "deterministic", - "failure_detail": "expected no tool calls for missing_parameter case, got ['base_tableList']", - "actual_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "" - } - } - ] - } - ] - }, - { - "case_id": "base_readQuery_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks for SQL, then calls base_readQuery after user provides the query", - "input": "Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'", - "expected_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'" - } - } - ], - "actual_output": "Great! Here are the results from your query:\n\n**Employees in the Sales Department:**\n\n| Name | Department |\n|------|------------|\n| Bob Smith | Sales |\n| Frank Lee | Sales |\n\n**Summary:** Found 2 employees in the Sales department.", - "metric_reasons": [], - "recommendation": null, - "turn_details": [ - { - "turn": 1, - "input": "Run a query for me", - "mode": "clarification", - "passed": true, - "actual_tools": [] - }, - { - "turn": 2, - "input": "Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'", - "mode": "tool", - "passed": true, - "actual_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'" - } - } - ] - } - ] - }, - { - "case_id": "base_columnDescription_happy", - "case_type": "happy_path", - "description": "Show basic column info for the evals employees table", - "input": "Show me the columns in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "actual_output": "The table **demo_user.evals_employees** contains the following **7 columns**:\n\n| Column Name | Data Type |\n|-------------|-----------|\n| **employee_id** | INTEGER |\n| **name** | VARCHAR |\n| **department** | VARCHAR |\n| **region** | VARCHAR |\n| **hire_date** | DATE |\n| **manager_id** | INTEGER |\n| **salary** | DECIMAL |\n\nThis appears to be an employee table with basic employee information including identification, organizational structure (department, region, manager), and compensation details.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_columnDescription_ambiguous_vs_columnMetadata", - "case_type": "ambiguous_selection", - "description": "A simple 'what columns does this table have' question should prefer base_columnDescription", - "input": "What are the column names and types in demo_user.evals_orders?", - "expected_tools": [ - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "actual_output": "The table `demo_user.evals_orders` contains the following columns:\n\n| Column Name | Data Type |\n|------------|-----------|\n| order_id | INTEGER |\n| order_date | DATE |\n| customer_name | VARCHAR |\n| product_category | VARCHAR |\n| quantity | INTEGER |\n| amount | DECIMAL |\n| ship_date | DATE |\n\nThe table has 7 columns in total, with a mix of date, numeric, and text fields suitable for order transaction data.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_columnMetadata_ambiguous_vs_columnDescription", - "case_type": "ambiguous_selection", - "description": "Asking for precise Teradata type codes and character sets should prefer base_columnMetadata", - "input": "Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_employees" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_employees" - } - } - ], - "actual_output": "Perfect! Here are the exact Teradata type codes, character sets, and precision details for every column in **demo_user.evals_employees**:\n\n## Column Metadata Summary\n\n| Column Name | Teradata Type Code | SQL Type String | Length | Character Set | Precision/Scale | Nullable | Index Type | Case Specificity |\n|-------------|-------------------|-----------------|---------|---------------|-----------------|----------|------------|------------------|\n| **employee_id** | **I** | INTEGER | 4 bytes | - | - | **NO** | **USI** (Unique Secondary Index) | NOT CASESPECIFIC |\n| **name** | **CV** | VARCHAR(100) LATIN | 100 chars | **LATIN** | - | **NO** | - | NOT CASESPECIFIC |\n| **department** | **CV** | VARCHAR(50) LATIN | 50 chars | **LATIN** | - | **NO** | - | NOT CASESPECIFIC |\n| **salary** | **D** | DOUBLE PRECISION | 8 bytes | - | Total: 10 digits, Fractional: 2 digits | **YES** | - | NOT CASESPECIFIC |\n| **region** | **CV** | VARCHAR(50) LATIN | 50 chars | **LATIN** | - | **NO** | - | NOT CASESPECIFIC |\n| **hire_date** | **DA** | DATE | 4 bytes | - | - | **NO** | - | NOT CASESPECIFIC |\n| **manager_id** | **I** | INTEGER | 4 bytes | - | - | **YES** | - | NOT CASESPECIFIC |\n\n### Key Details:\n\n- **Teradata Type Codes:**\n - `I` = INTEGER (4-byte signed integer)\n - `CV` = CHARACTER VARYING (VARCHAR)\n - `D` = DOUBLE PRECISION (floating-point)\n - `DA` = DATE\n\n- **Character Sets:** All string columns use **LATIN** character set (CharType=1)\n\n- **Indexes:** `employee_id` has a **Unique Secondary Index (USI)**\n\n- **Nullable Columns:** Only `salary` and `manager_id` allow NULL values\n\n- **Formats:**\n - Integers: `-(10)9` (10-digit signed format)\n - Strings: `X(n)` where n is the length\n - Salary: `---------.99` (decimal with 2 fractional digits)\n - Date: `YY/MM/DD`", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "base_unknown_tool_missing", - "case_type": "missing_parameter", - "description": "User asks to do something no tool supports \u2014 agent should explain and ask for clarification", - "input": "Automatically tune my Teradata database indexes and rebalance the AMPs", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I appreciate your interest in optimizing your Teradata database, but I need to clarify what's possible with the available tools and Teradata's architecture.\n\n## Current Tool Capabilities\n\nThe tools I have access to are **read-only analysis and monitoring tools**. They can:\n\n- \u2705 Analyze table usage patterns and query affinity\n- \u2705 Identify space usage and performance bottlenecks\n- \u2705 Review query execution history and resource consumption\n- \u2705 Examine table structures, indexes, and DDL\n- \u2705 Detect data quality issues\n- \u2705 Analyze SQL query patterns for optimization opportunities\n\n## What I Cannot Do Automatically\n\nI **cannot** execute DDL/DML operations such as:\n\n- \u274c Creating, dropping, or modifying indexes\n- \u274c Rebalancing AMPs or data distribution\n- \u274c Altering table structures\n- \u274c Changing primary indexes or partitioning\n\n## What I Can Help You With\n\nInstead, I can help you make **informed decisions** about index optimization:\n\n1. **Analyze Query Patterns**: Identify which tables are most frequently accessed and how they're queried\n2. **Review Current Indexes**: Examine existing table structures and index definitions\n3. **Identify Optimization Opportunities**: Find tables with performance issues, skew problems, or missing indexes\n4. **Generate Recommendations**: Suggest specific DDL statements you can review and execute\n\n## AMP Rebalancing Note\n\nTeradata **automatically handles AMP rebalancing** as part of its architecture. Data distribution is determined by the Primary Index (PI) hash. If you're experiencing skew issues, the solution is typically to:\n- Choose a better Primary Index column\n- Use a composite Primary Index\n- Consider partitioning strategies\n\n---\n\n**Would you like me to:**\n1. Analyze your current table usage patterns to identify optimization candidates?\n2. Review specific tables for potential index improvements?\n3. Examine query performance to find tables that would benefit from secondary indexes?\n\nPlease let me know which database you'd like me to analyze, and I'll provide actionable recommendations with the specific DDL you would need to execute.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "chat_completeChat_missing_message", - "case_type": "missing_parameter", - "description": "User invokes chat without providing a message", - "input": "Can you use the chat completion tool for me?", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I don't have a chat completion tool available in my current set of tools. The tools I have access to are primarily focused on:\n\n1. **Database operations** - querying Teradata databases, listing tables, viewing table structures\n2. **Data quality analysis** - checking for missing values, duplicates, statistics\n3. **Database administration** - monitoring space usage, user activity, sessions\n4. **Security** - checking user permissions and roles\n5. **SQL optimization** - clustering and analyzing SQL queries\n6. **Graph analysis** - analyzing database dependencies and lineage\n7. **RAG (Retrieval-Augmented Generation)** - answering questions based on document context\n8. **Visualization** - creating charts from data\n\nIf you're looking to have a conversation or need help with text generation, I can assist directly without needing a separate chat completion tool. \n\nHow can I help you? Are you looking to:\n- Work with a Teradata database?\n- Analyze data quality?\n- Visualize some data?\n- Something else?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "chat_unknown_tool_missing", - "case_type": "missing_parameter", - "description": "User asks for a non\u2011existent chat tool", - "input": "Talk to the assistant about budgeting", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you with budgeting! However, I should clarify my role and capabilities.\n\nI'm a specialized assistant designed to work with **Teradata databases**. My tools are focused on:\n\n- **Database Administration**: Analyzing disk space, table usage, system resources\n- **Data Quality**: Checking for missing values, statistics, data profiling\n- **Security**: Managing user permissions, roles, and access rights\n- **Query Analysis**: Examining SQL execution patterns, performance optimization\n- **Graph Analytics**: Dependency analysis and lineage tracing\n- **RAG (Retrieval Augmented Generation)**: Document-based question answering\n\nIf you're looking to discuss **personal or business budgeting** (managing finances, expenses, savings, etc.), I wouldn't be the right assistant for that topic, as I don't have tools designed for financial planning.\n\nHowever, if you're interested in budgeting related to:\n- **Database resource budgeting** (storage space allocation, capacity planning)\n- **Query cost analysis** (CPU/IO resource consumption)\n- **Workload management** (understanding resource usage patterns)\n\nThen I can definitely help! Could you let me know which type of budgeting you're interested in discussing?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_tableSpace_happy", - "case_type": "happy_path", - "description": "List largest tables by storage in the evals database", - "input": "Show me the 10 largest tables in demo_user ranked by storage consumption.", - "expected_tools": [ - { - "name": "dba_tableSpace", - "params": { - "database_name": "demo_user", - "top_n": 10 - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_tableSpace", - "params": { - "database_name": "demo_user", - "top_n": 10 - } - } - ], - "actual_output": "Here are the **10 largest tables in demo_user** ranked by storage consumption:\n\n| Rank | Database Name | Table Name | Current Storage (bytes) | Peak Storage (bytes) | Skew % |\n|------|---------------|------------|------------------------|---------------------|--------|\n| 1 | demo_user | **All** (Total) | 1,146,880 | 1,146,880 | 16.67% |\n| 2 | demo_user | **get_data** | 294,912 | 294,912 | 35.71% |\n| 3 | demo_user | **remove_data** | 262,144 | 262,144 | 33.33% |\n| 4 | demo_user | **evals_orders** | 196,608 | 196,608 | 0.00% |\n| 5 | demo_user | **space_report** | 196,608 | 196,608 | 25.00% |\n| 6 | demo_user | **evals_employees** | 196,608 | 196,608 | 0.00% |\n\n**Key Insights:**\n- The database contains only **5 tables** (the \"All\" row is an aggregate total)\n- **get_data** is the largest table at ~288 KB, consuming about 25.7% of total space\n- **remove_data** is second at ~256 KB (22.9% of total space)\n- The remaining three tables are roughly equal in size at ~192 KB each\n- **get_data** and **remove_data** show higher data skew (35.71% and 33.33%), suggesting uneven distribution across AMPs\n- **evals_orders** and **evals_employees** show perfect distribution with 0% skew\n\nTotal database storage: **~1.1 MB**", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_tableSqlList_happy", - "case_type": "happy_path", - "description": "List SQL executed against evals_orders over the past 14 days", - "input": "Show me all queries run against evals_orders over the last 14 days.", - "expected_tools": [ - { - "name": "dba_tableSqlList", - "params": { - "table_name": "evals_orders", - "no_days": 14 - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_tableSqlList", - "params": { - "table_name": "evals_orders", - "no_days": 14 - } - } - ], - "actual_output": "Here are all the queries run against the **evals_orders** table over the last 14 days:\n\n## Summary\nFound **16 queries** executed by **DEMO_USER** on **June 23, 2026**\n\n## Query Types\n\n### 1. **Data Retrieval Queries**\n- `SELECT * FROM demo_user.evals_orders WHERE amount > 500` (2 occurrences)\n- `SELECT TOP 5 * FROM demo_user.evals_orders` (4 occurrences)\n- `SELECT COUNT(*) FROM demo_user.evals_orders` (2 occurrences)\n\n### 2. **Metadata Queries**\n- `SHOW TABLE demo_user.evals_orders` (5 occurrences)\n\n### 3. **Table Affinity Analysis**\n- Complex query analyzing which tables are queried together with evals_orders (3 occurrences)\n - This query joins `DBC.DBQLObjTbl` with `DBC.DBQLogTbl` to find co-occurring tables in queries\n\n## Key Observations\n- All queries were executed by the **DEMO_USER** account\n- Most activity occurred on **June 23, 2026** around **20:38 - 21:17**\n- The most common operations were:\n - Previewing the table structure (`SHOW TABLE`)\n - Sampling data (`TOP 5`)\n - Filtering records by amount (`WHERE amount > 500`)\n - Analyzing table relationships (affinity analysis)\n\nAll queries appear to be exploratory in nature, focused on understanding the table structure and its relationships with other tables in the database.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_userSqlList_happy", - "case_type": "happy_path", - "description": "Show queries executed by a specific user over the past 14 days", - "input": "Show me what queries John_Smith has been executing over the past 14 days.", - "expected_tools": [ - { - "name": "dba_userSqlList", - "params": { - "user_name": "John_Smith", - "no_days": 14 - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_userSqlList", - "params": { - "user_name": "John_Smith", - "no_days": 14 - } - } - ], - "actual_output": "Based on the query results, **no SQL queries were found** for user `John_Smith` in the past 14 days.\n\nThis could mean:\n- The user hasn't executed any queries during this period\n- The username might be slightly different (case-sensitive or with different formatting)\n- The query logging may not have captured activity for this user\n- The DBQL (Database Query Logging) data may have been purged or not enabled for this user\n\nWould you like me to:\n1. Check if there are any sessions currently active for John_Smith?\n2. Try searching with a different username variation?\n3. Look at a longer time period?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_databaseSpace_happy", - "case_type": "happy_path", - "description": "Storage allocated to the evals database", - "input": "How much disk space is allocated to the demo_user database?", - "expected_tools": [ - { - "name": "dba_databaseSpace", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_databaseSpace", - "params": { - "database_name": "demo_user" - } - } - ], - "actual_output": "The **demo_user** database has the following disk space allocation:\n\n- **Space Allocated**: 78.0 GB\n- **Space Used**: 0.0 GB\n- **Free Space**: 78.0 GB\n- **Percent Used**: 0.0%\n\nThe database has 78 GB of space allocated to it, and currently none of that space is being used - it's completely empty.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_tableUsageImpact_happy", - "case_type": "happy_path", - "description": "Which users are consuming the most resources in the evals database", - "input": "Which users are consuming the most resources in demo_user?", - "expected_tools": [ - { - "name": "dba_tableUsageImpact", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_tableUsageImpact", - "params": { - "database_name": "demo_user" - } - }, - { - "name": "dba_resusageSummary", - "params": { - "no_days": 30 - } - } - ], - "actual_output": "Perfect! Based on the system-wide resource usage data, I can see that **DEMO_USER** is the primary user consuming resources. Here's a summary of their resource consumption over the last 30 days:\n\n## Resource Consumption by DEMO_USER\n\n### Summary Statistics:\n- **Primary User**: DEMO_USER (most active)\n- **Secondary User**: DBC (minimal system activity)\n\n### DEMO_USER Activity Breakdown:\n\n**Top Resource-Consuming Periods:**\n\n1. **June 19, 1:00 AM (Friday)** - Highest CPU Usage\n - 147 queries\n - **24.20 CPU seconds**\n - 3.71 GB I/O\n - Workload: Query/Answers via PYTHON3\n\n2. **June 15, 6:00 PM (Monday)**\n - 366 queries\n - **12.32 CPU seconds**\n - 3.23 GB I/O\n - Workload: Query/Answers via PYTHON3\n\n3. **June 23, 7:00 PM (Tuesday)**\n - 64 queries\n - **5.73 CPU seconds**\n - 1.97 GB I/O\n - Workload: Query/Answers via PYTHON3\n\n4. **June 8, 5:00 PM (Monday)** - ETL Activity\n - 1,957 requests\n - **3.17 CPU seconds**\n - 0.64 GB I/O\n - Workload: ETL/ELT via BTEQ\n\n### Key Insights:\n- **Application Tools**: Primarily PYTHON3, some BTEQ for ETL\n- **Workload Types**: Mostly \"Answers\" complexity queries and some ETL/System operations\n- **Pattern**: Activity spans multiple days with peak usage during evening hours (5-7 PM) and some overnight processing (1 AM)\n- **DEMO_USER** is essentially the only active user in the system (aside from minimal DBC system activity)\n\nWould you like me to drill down into specific time periods or analyze the types of queries being run?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_resusageSummary_happy", - "case_type": "happy_path", - "description": "System-wide CPU/memory/IO by application over the past month", - "input": "Show me how much CPU, memory, and I/O was consumed over the past month, grouped by application.", - "expected_tools": [ - { - "name": "dba_resusageSummary", - "params": { - "no_days": 30 - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_resusageSummary", - "params": { - "no_days": 30 - } - } - ], - "actual_output": "Based on the resource usage data from the past 30 days, here's a summary of CPU, memory, and I/O consumption grouped by application:\n\n## **Resource Consumption by Application (Past 30 Days)**\n\n### **PYTHON3 (Primary Application)**\n- **Request Count**: 1,495+ requests\n- **Total CPU Time**: 46+ seconds\n- **Total I/O Operations**: 234,546+ logical I/O\n- **Total I/O Volume**: ~15+ GB logical I/O\n- **Physical I/O**: ~132 operations, ~8 MB physical I/O\n- **Workload Types**: \n - QUERY (Answers) - majority\n - ETL/ELT (System/Procedural)\n - OTHER (System/Procedural)\n\n### **BTEQ (Batch Processing)**\n- **Request Count**: 1,957 requests\n- **Total CPU Time**: 3.17 seconds\n- **Total I/O Operations**: 19,923 logical I/O\n- **Total I/O Volume**: 0.64 GB logical I/O\n- **Physical I/O**: 277 operations, ~4.5 MB physical I/O\n- **Workload Type**: ETL/ELT (System/Procedural)\n- **Peak Usage**: June 8th at 5 PM\n\n### **NODE (System Process)**\n- **Request Count**: 2 requests\n- **Total CPU Time**: 0.024 seconds\n- **Total I/O Operations**: 20 logical I/O\n- **Total I/O Volume**: 0.0006 GB logical I/O\n- **User**: DBC (system user)\n- **Workload Type**: OTHER (System/Procedural)\n\n### **Key Insights:**\n1. **PYTHON3 dominates resource usage** - accounting for the vast majority of CPU, I/O, and request volume\n2. **Peak hours**: Generally between 5-7 PM on weekdays\n3. **Workload pattern**: Most activity is analytical queries (\"Answers\" complexity)\n4. **I/O efficiency**: Low physical I/O relative to logical I/O indicates good caching\n\nWould you like me to drill down into any specific application, time period, or workload type?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_databaseVersion_happy", - "case_type": "happy_path", - "description": "Get current Teradata software version", - "input": "What release of Teradata software is currently running on our system?", - "expected_tools": [ - { - "name": "dba_databaseVersion", - "params": {} - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_databaseVersion", - "params": {} - } - ], - "actual_output": "Your Teradata system is currently running:\n\n- **Version/Release**: 20.00.24.60\n- **Language Support Mode**: Standard\n\nThis is Teradata Database version 20.00, which is part of the Teradata Vantage platform. The build/maintenance level is 24.60.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_flowControl_happy", - "case_type": "happy_path", - "description": "System throttling and delay statistics for a date range", - "input": "Show me the system throttling and flow control delay statistics from January 15th 2024 through February 28th 2024.", - "expected_tools": [ - { - "name": "dba_flowControl", - "params": { - "start_date": "2024-01-15", - "end_date": "2024-02-28" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_flowControl", - "params": { - "start_date": "2024-01-15", - "end_date": "2024-02-28" - } - } - ], - "actual_output": "Based on the query results, **no flow control events or throttling delays were recorded** during the period from January 15th, 2024 through February 28th, 2024.\n\nThis indicates that:\n\n- The Teradata workload management system did not impose any flow control restrictions during this timeframe\n- There were no resource constraint conditions that required the system to throttle or delay queries\n- The system had sufficient capacity to handle the workload without triggering flow control mechanisms\n\n**What this means:**\nFlow control events typically occur when the system experiences resource pressure (CPU, I/O, memory, or AMP worker task exhaustion). The absence of flow control events suggests the system was operating within its capacity limits throughout this period.\n\nIf you're investigating performance issues during this timeframe, you may want to look at:\n- User-level query delays using the **dba_userDelay** tool to see if queries were waiting in queues\n- Resource consumption patterns using **dba_resusageSummary** to understand overall system utilization\n- Specific query performance from the query logs\n\nWould you like me to investigate any of these alternative performance metrics?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_featureUsage_happy", - "case_type": "happy_path", - "description": "Feature utilization metrics for a date range", - "input": "Show me how different Teradata features were being utilized by our users between January 15th, 2024 and March 31st, 2024.", - "expected_tools": [ - { - "name": "dba_featureUsage", - "params": { - "start_date": "2024-01-15", - "end_date": "2024-03-31" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_featureUsage", - "params": { - "start_date": "2024-01-15", - "end_date": "2024-03-31" - } - } - ], - "actual_output": "Based on the query results, **no Teradata feature usage data was found** for the period between January 15th, 2024 and March 31st, 2024.\n\nThis could mean:\n\n1. **No query logs available**: The DBC.DBQLogTbl (query logging table) may not contain data for this time period\n2. **Query logging not enabled**: Feature usage tracking requires query logging to be enabled on the Teradata system\n3. **Logs aged out**: Historical query logs may have been purged before this date range\n4. **No queries executed**: No user queries were run during this period (unlikely)\n\n**To investigate further, you could:**\n\n- Check what date ranges ARE available in the query logs\n- Verify that query logging is enabled on your Teradata system\n- Check with your DBA about log retention policies\n- Try a more recent date range to see if current data is being captured\n\nWould you like me to check what query log data is available or investigate a different time period?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_userDelay_happy", - "case_type": "happy_path", - "description": "How long users waited in queues for a date range", - "input": "Show me how long users were waiting in the query queue between March 1st 2024 and March 31st 2024.", - "expected_tools": [ - { - "name": "dba_userDelay", - "params": { - "start_date": "2024-03-01", - "end_date": "2024-03-31" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_userDelay", - "params": { - "start_date": "2024-03-01", - "end_date": "2024-03-31" - } - } - ], - "actual_output": "Based on the query results for March 1-31, 2024, **there were no queries that experienced any queue delays during this period**. \n\nThe query specifically looks for instances where `DelayTime > 0`, meaning queries that had to wait in the queue before execution. Since no records were returned, this indicates that:\n\n- All queries during March 2024 were able to execute immediately without waiting\n- The system had sufficient resources available to handle the workload\n- No workload management throttling or queueing occurred during this time period\n\nThis is generally a positive indicator of system performance, suggesting that the Teradata system had adequate capacity to handle the query workload during March 2024 without requiring users to wait in queues.\n\nIf you'd like to:\n- Check a different time period that might have had more activity\n- Look at system-level flow control events (which would show throttling at the workload management level)\n- Investigate resource usage patterns during this period\n\nJust let me know!", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_sessionInfo_happy", - "case_type": "happy_path", - "description": "Active sessions for a specific user", - "input": "Show me what active connections the database user SALES_ADMIN currently has open in Teradata.", - "expected_tools": [ - { - "name": "dba_sessionInfo", - "params": { - "user_name": "SALES_ADMIN" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_sessionInfo", - "params": { - "user_name": "SALES_ADMIN" - } - } - ], - "actual_output": "The query returned no results, which means the database user **SALES_ADMIN** currently has **no active connections** open in Teradata.\n\nThis could mean:\n- The user is not currently logged in\n- All sessions for this user have been closed\n- The user account exists but has no active sessions at this moment\n\nIf you were expecting to see active sessions, you may want to verify:\n- The exact username spelling (it's case-insensitive in the query)\n- Whether the user has recently disconnected\n- If there are any connection issues preventing login", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_systemSpace_happy", - "case_type": "happy_path", - "description": "Total disk consumption across all databases", - "input": "How much disk capacity is currently being consumed across all databases in our data warehouse?", - "expected_tools": [ - { - "name": "dba_systemSpace", - "params": {} - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_systemSpace", - "params": {} - } - ], - "actual_output": "## Data Warehouse Disk Capacity Summary\n\nHere's the current disk space consumption across all databases in your data warehouse:\n\n| Metric | Value |\n|--------|-------|\n| **Space Used** | 0 GB |\n| **Space Allocated** | 92 GB |\n| **Percentage Used** | 0% |\n| **Free Space** | 92 GB |\n\n### Key Findings:\n\n- **Total Allocated Capacity**: 92 GB has been allocated across all databases\n- **Current Consumption**: Effectively 0 GB is currently being used (less than 1 GB)\n- **Available Space**: 92 GB remains available for use\n- **Utilization Rate**: 0% - your data warehouse is essentially empty or has minimal data\n\nThis indicates that your data warehouse has plenty of available capacity and is currently consuming very little disk space. If you'd like to see which specific databases or tables are using space, I can drill down into:\n- Space usage by individual database using `dba_databaseSpace`\n- Space usage by tables within a specific database using `dba_tableSpace`", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_tableSpace_ambiguous_vs_databaseSpace", - "case_type": "ambiguous_selection", - "description": "Ranking tables by size inside one database should prefer dba_tableSpace, not dba_databaseSpace", - "input": "Which tables in demo_user are using the most disk space?", - "expected_tools": [ - { - "name": "dba_tableSpace", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_tableSpace", - "params": { - "database_name": "demo_user" - } - } - ], - "actual_output": "Here are the tables in the **demo_user** database ranked by disk space usage:\n\n| **Table Name** | **Current Space (bytes)** | **Peak Space (bytes)** | **Skew %** |\n|---|---|---|---|\n| **get_data** | 294,912 | 294,912 | 35.71% |\n| **remove_data** | 262,144 | 262,144 | 33.33% |\n| **evals_orders** | 196,608 | 196,608 | 0.0% |\n| **space_report** | 196,608 | 196,608 | 25.0% |\n| **evals_employees** | 196,608 | 196,608 | 0.0% |\n\n**Key Findings:**\n- **Total database space:** 1,146,880 bytes (~1.09 MB)\n- **Largest table:** `get_data` uses 294,912 bytes (~288 KB), representing about 26% of the database\n- **Second largest:** `remove_data` uses 262,144 bytes (~256 KB), about 23% of the database\n- **Data skew:** Both `get_data` (35.71%) and `remove_data` (33.33%) show significant data skew across AMPs, while `evals_orders` and `evals_employees` have perfect distribution (0% skew)\n\nThe top two tables (`get_data` and `remove_data`) account for nearly half of the database's disk usage.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_databaseSpace_ambiguous_vs_systemSpace", - "case_type": "ambiguous_selection", - "description": "Storage for one named database should prefer dba_databaseSpace, not dba_systemSpace", - "input": "How much total space is the demo_user database using?", - "expected_tools": [ - { - "name": "dba_databaseSpace", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_databaseSpace", - "params": { - "database_name": "demo_user" - } - } - ], - "actual_output": "The **demo_user** database has the following disk space usage:\n\n- **Space Allocated**: 78.0 GB\n- **Space Used**: 0.0 GB\n- **Free Space**: 78.0 GB\n- **Percent Used**: 0.0%\n\nThe database currently has 78 GB of space allocated to it but is not using any of that space yet (0.0 GB used). This means the database exists but either has no tables in it or the tables are empty.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_systemSpace_ambiguous_vs_databaseSpace", - "case_type": "ambiguous_selection", - "description": "Warehouse-wide storage across all databases should prefer dba_systemSpace, not dba_databaseSpace", - "input": "What is the total disk usage across the entire Teradata system?", - "expected_tools": [ - { - "name": "dba_systemSpace", - "params": {} - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_systemSpace", - "params": {} - } - ], - "actual_output": "Based on the system-wide disk space analysis, here's the total disk usage across your entire Teradata system:\n\n## System Disk Space Summary\n\n- **Total Space Allocated**: 92 GB\n- **Space Currently Used**: 0 GB\n- **Percentage Used**: 0%\n- **Free Space Available**: 92 GB\n\nThe system currently has 92 GB of total allocated disk space with essentially no space being used at this time. This indicates either a new system or one that has been recently cleaned/reset with minimal data loaded.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_tableSqlList_ambiguous_vs_userSqlList", - "case_type": "ambiguous_selection", - "description": "Queries run against a table should prefer dba_tableSqlList, not dba_userSqlList", - "input": "What SQL has been executed against evals_orders in the past week?", - "expected_tools": [ - { - "name": "dba_tableSqlList", - "params": { - "table_name": "evals_orders", - "no_days": 7 - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_tableSqlList", - "params": { - "table_name": "evals_orders", - "no_days": 7 - } - } - ], - "actual_output": "Based on the query log for the `evals_orders` table over the past week, here are the SQL statements that have been executed:\n\n## Summary of SQL Activity\n\n**User:** DEMO_USER \n**Time Period:** Past 7 days (mostly from June 23, 2026)\n\n### Main Query Types:\n\n1. **Table Preview Queries:**\n - `SELECT TOP 5 * FROM demo_user.evals_orders`\n - Multiple executions to sample the table data\n\n2. **Filtered Data Retrieval:**\n - `SELECT * FROM demo_user.evals_orders WHERE amount > 500`\n - Executed multiple times to retrieve orders with amounts greater than 500\n\n3. **Count Queries:**\n - `SELECT COUNT(*) FROM demo_user.evals_orders`\n - Used to get total row counts\n\n4. **Table Metadata:**\n - `SHOW TABLE demo_user.evals_orders`\n - Multiple executions to view table structure\n\n5. **Table Affinity Analysis:**\n - Complex query joining `DBC.DBQLObjTbl` and `DBC.DBQLogTbl` to analyze which tables are queried together with `evals_orders`\n - This appears to be an internal system query for understanding table relationships\n\n### Key Observations:\n\n- All queries were executed by the **DEMO_USER** account\n- Most activity occurred on **June 23, 2026** between 20:38 and 21:17\n- The queries are primarily **read operations** (SELECT statements)\n- No INSERT, UPDATE, DELETE, or DDL operations were detected\n- Common patterns include data sampling, filtering, and metadata inspection\n\nThe table appears to be used mainly for data exploration and analysis purposes during this period.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_userSqlList_ambiguous_vs_tableSqlList", - "case_type": "ambiguous_selection", - "description": "Queries run by a specific user should prefer dba_userSqlList, not dba_tableSqlList", - "input": "List all SQL statements that user ANALYTICS_USER has run over the past 7 days.", - "expected_tools": [ - { - "name": "dba_userSqlList", - "params": { - "user_name": "ANALYTICS_USER", - "no_days": 7 - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_userSqlList", - "params": { - "user_name": "ANALYTICS_USER", - "no_days": 7 - } - } - ], - "actual_output": "The query returned **no SQL statements** for user ANALYTICS_USER over the past 7 days.\n\nThis could mean:\n1. **The user has not run any queries** in the past 7 days\n2. **Query logging is not enabled** or not capturing data for this user\n3. **The user name may be incorrect** - Teradata user names are case-sensitive\n4. **Query log data may have been purged** or is not being retained\n\n**Suggestions:**\n- Verify the exact user name (check capitalization)\n- Check if query logging (DBQL) is enabled on the system\n- Try expanding the time range beyond 7 days to see if historical data exists\n- Verify that the user actually has been running queries during this period\n\nWould you like me to:\n- Check with a different user name?\n- Extend the time range (e.g., 30 days)?\n- List all active users who have run queries recently?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_tableUsageImpact_ambiguous_vs_resusageSummary", - "case_type": "ambiguous_selection", - "description": "Heavy users of a specific database should prefer dba_tableUsageImpact, not dba_resusageSummary", - "input": "Who is generating the heaviest workload against demo_user?", - "expected_tools": [ - { - "name": "dba_tableUsageImpact", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_tableUsageImpact", - "params": { - "database_name": "demo_user" - } - } - ], - "actual_output": "The query returned no results, which means there's no recorded workload activity against the **demo_user** database in the DBQL (Database Query Log) tables.\n\nThis could happen for several reasons:\n\n1. **DBQL logging may not be enabled** for queries against this database\n2. **No recent query activity** has occurred against demo_user tables\n3. **The database may be empty** or not actively used\n4. **DBQL retention period** may have expired and historical data has been purged\n\nTo investigate further, I can:\n- Check if the demo_user database exists and contains tables\n- Look at system-wide resource usage to see if there's activity elsewhere\n- Check session information to see if there are active connections\n\nWould you like me to check if the demo_user database exists and what tables it contains?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_resusageSummary_ambiguous_vs_tableUsageImpact", - "case_type": "ambiguous_selection", - "description": "System-wide CPU/memory/I/O by application should prefer dba_resusageSummary, not dba_tableUsageImpact", - "input": "Break down CPU, memory, and I/O consumption by application over the last 30 days.", - "expected_tools": [ - { - "name": "dba_resusageSummary", - "params": { - "no_days": 30 - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_resusageSummary", - "params": { - "no_days": 30 - } - } - ], - "actual_output": "Based on the resource usage data over the last 30 days, here's the breakdown of CPU, memory, and I/O consumption by application:\n\n## **Resource Consumption by Application (Last 30 Days)**\n\n### **Primary Applications Identified:**\n\n#### **1. PYTHON3 (Dominant Application)**\n- **Request Count:** 577 total requests across various workloads\n- **CPU Usage:** \n - Peak session: 24.20 seconds (AMPCPUTime) on June 19\n - Average range: 1.40 - 12.32 seconds per session\n- **I/O Consumption:**\n - Peak logical I/O: **85,309 operations** (3.71 GB)\n - Typical range: 7,547 - 85,309 I/O operations\n - Physical I/O: Minimal (0 - 71 operations), indicating good caching\n- **Workload Types:**\n - Query/Answers (primary)\n - ETL/ELT\n - System/Procedural\n\n#### **2. BTEQ (Batch Processing)**\n- **Request Count:** 1,957 requests\n- **CPU Usage:** 3.17 seconds (AMPCPUTime)\n- **I/O Consumption:**\n - Logical I/O: 19,923 operations (0.64 GB)\n - Physical I/O: 277 operations (4.53 MB)\n- **Workload Type:** ETL/ELT System/Procedural\n\n#### **3. NODE (System Operations)**\n- **Request Count:** 2 requests (minimal activity)\n- **CPU Usage:** 0.024 seconds\n- **I/O Consumption:** 20 operations (0.56 MB)\n- **Workload Type:** System/Procedural\n\n### **Key Insights:**\n\n1. **PYTHON3 dominates resource consumption** - responsible for the majority of query workloads and highest CPU/I/O usage\n2. **Excellent cache efficiency** - Very low physical I/O compared to logical I/O indicates data is mostly served from cache\n3. **Peak activity periods:**\n - Monday evenings (18:00-19:00)\n - Friday early morning (01:00)\n - Tuesday evening (19:00)\n4. **Workload complexity:** Most queries classified as \"Answers\" complexity with some ETL/ELT procedural work\n\n**Note:** The data shows Total Server Byte Count as 0 across all entries, indicating memory metrics may not be captured in this particular logging configuration. CPU and I/O metrics are fully available and detailed above.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_tableUsageImpact_ambiguous_vs_sessionInfo", - "case_type": "ambiguous_selection", - "description": "Historical resource consumption by users in a database should prefer dba_tableUsageImpact, not dba_sessionInfo", - "input": "Which users have been consuming the most CPU and query resources in demo_user over time?", - "expected_tools": [ - { - "name": "dba_tableUsageImpact", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_tableUsageImpact", - "params": { - "database_name": "demo_user" - } - }, - { - "name": "dba_resusageSummary", - "params": { - "no_days": 30 - } - } - ], - "actual_output": "Perfect! Based on the resource usage data over the past 30 days, here's what I found about CPU and query resource consumption:\n\n## **CPU and Query Resource Consumption Summary**\n\n### **Top User: DEMO_USER**\n\nThe **DEMO_USER** account has been the dominant consumer of system resources. Here are the key findings:\n\n### **Highest Activity Periods:**\n\n1. **June 19 at 1:00 AM (Friday)** - Peak CPU usage\n - **24.20 CPU seconds** across 147 queries\n - 3.71 GB of I/O processed\n - Workload type: QUERY (Answers complexity)\n\n2. **June 15 at 6:00 PM (Monday)** - High volume\n - **12.32 CPU seconds** across 366 queries\n - 3.23 GB of I/O processed\n - Application: PYTHON3\n\n3. **June 23 at 7:00 PM (Tuesday)**\n - **5.73 CPU seconds** across 64 queries\n - 1.97 GB of I/O processed\n\n### **Workload Breakdown:**\n\n- **Query Workload (Answers)**: Most CPU-intensive, typically running analytical queries via PYTHON3\n- **ETL/ELT Operations**: Lower CPU but involved in data loading (using BTEQ on June 8)\n- **System/Procedural**: Administrative tasks with minimal resource consumption\n\n### **Secondary User:**\n- **DBC** (system account): Minimal usage - 0.024 CPU seconds on June 8\n\n### **Key Observations:**\n\n1. **DEMO_USER is the sole significant user** consuming resources in this system\n2. **Python-based applications** are the primary access method\n3. **Peak activity occurs during evening hours** (6-7 PM) and early morning (1 AM)\n4. **Monday and Friday** show the highest activity levels\n5. Total query count ranges from **5 to 366 queries** per session\n\nWould you like me to drill deeper into specific time periods or analyze the SQL queries being executed by DEMO_USER?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_sessionInfo_ambiguous_vs_tableUsageImpact", - "case_type": "ambiguous_selection", - "description": "Open sessions for a specific user should prefer dba_sessionInfo, not dba_tableUsageImpact", - "input": "What active sessions does user SALES_ADMIN currently have open?", - "expected_tools": [ - { - "name": "dba_sessionInfo", - "params": { - "user_name": "SALES_ADMIN" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_sessionInfo", - "params": { - "user_name": "SALES_ADMIN" - } - } - ], - "actual_output": "The user **SALES_ADMIN** currently has **no active sessions** open on the Teradata system.\n\nThis means:\n- The user is not currently logged in\n- There are no open connections or database sessions associated with this user account\n- No queries or operations are being executed under this user's credentials at this time\n\nIf you were expecting to see active sessions, the user may have logged out, their sessions may have timed out, or they simply aren't connected at the moment.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_userDelay_ambiguous_vs_flowControl", - "case_type": "ambiguous_selection", - "description": "How long users waited in queues should prefer dba_userDelay, not dba_flowControl", - "input": "How long were users waiting in the query queue between March 1 and March 31, 2024?", - "expected_tools": [ - { - "name": "dba_userDelay", - "params": { - "start_date": "2024-03-01", - "end_date": "2024-03-31" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_userDelay", - "params": { - "start_date": "2024-03-01", - "end_date": "2024-03-31" - } - } - ], - "actual_output": "Based on the query results, **there were no user delays in the query queue between March 1 and March 31, 2024**. \n\nThe system found zero instances where users had to wait (DelayTime > 0) before their queries started executing during this period. This indicates that:\n\n- **No queries experienced queue delays** - All queries were able to start executing immediately\n- **The system had sufficient capacity** - There were no resource constraints causing queries to wait\n- **Workload management was effective** - No throttling or queuing was necessary\n\nThis is a positive indicator of system performance, suggesting that the Teradata system had adequate resources to handle the workload during March 2024 without requiring users to wait in the query queue.\n\nIf you'd like to check:\n- System-wide throttling and flow control events (even without user delays), I can use the `dba_flowControl` tool\n- Resource usage patterns during this period to understand overall system load\n- A different time period to see if delays occurred at other times\n\nWould you like me to investigate any of these aspects?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_flowControl_ambiguous_vs_userDelay", - "case_type": "ambiguous_selection", - "description": "System throttling and flow-control statistics should prefer dba_flowControl, not dba_userDelay", - "input": "Show throttling and flow-control delay statistics from January 15 through February 28, 2024.", - "expected_tools": [ - { - "name": "dba_flowControl", - "params": { - "start_date": "2024-01-15", - "end_date": "2024-02-28" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_flowControl", - "params": { - "start_date": "2024-01-15", - "end_date": "2024-02-28" - } - } - ], - "actual_output": "## Throttling and Flow-Control Statistics\n**Period: January 15 - February 28, 2024**\n\n**No flow control events were recorded during this period.**\n\nThe query successfully executed against the Teradata workload management tables (DBC.RESUSAGESAWT, DBC.RESUSAGESVPR, DBC.RESUSAGESPMA), but returned zero rows. This indicates that:\n\n- **No throttling occurred**: The system did not impose any workload management flow control restrictions during this timeframe\n- **No resource-based delays**: Queries were not delayed due to CPU, memory, or I/O resource constraints\n- **Healthy system state**: The absence of flow control events suggests the system had adequate capacity to handle the workload without throttling\n\n**What the report would have shown (if events existed):**\n- Date and time of each flow control event\n- Duration of flow control delays\n- FlowControl% - percentage of time queries were throttled\n- CPU utilization metrics (execution, service, I/O wait, idle percentages)\n- Correlation between CPU pressure and throttling events\n\nThis is generally a positive indicator of system health during the specified period.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_tableSpace_missing_db", - "case_type": "missing_parameter", - "description": "Agent must ask which database to list biggest tables for", - "input": "Show me the biggest tables", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you see the biggest tables! However, I need to know which database you'd like to check. \n\nCould you please specify which database you want to see the largest tables for?\n\nFor example:\n- A specific database name (e.g., \"SALES_DB\", \"DEV01_StGeo_STD_T\")\n- Or if you'd like to see system-wide space usage across all databases", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_tableSqlList_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask which table to list queries for", - "input": "Show me the queries from the last 14 days", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to show you the queries from the last 14 days, but I need a bit more information to give you the most relevant results.\n\nCould you please clarify what type of query history you're looking for?\n\n1. **Queries run by a specific user** - If you want to see queries executed by a particular user account, I can use the user SQL history tool. Please provide the username.\n\n2. **Queries run against a specific table** - If you want to see queries that accessed a particular table, I can use the table SQL history tool. Please provide the table name.\n\n3. **System-wide query activity** - If you want broader system metrics like resource usage, workload patterns, or user activity across the system, I can provide that as well.\n\nWhich would you prefer, and what specific details should I filter by?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_userSqlList_missing_user", - "case_type": "missing_parameter", - "description": "Agent must ask which user to list queries for", - "input": "Show me the queries a user executed over the past 14 days", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you retrieve the SQL queries executed by a user over the past 14 days. However, I need to know which specific user account you'd like to analyze.\n\nCould you please provide the username (Teradata user account name) you want to investigate?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_databaseSpace_missing_db", - "case_type": "missing_parameter", - "description": "Agent must ask which database to get storage for", - "input": "How much space is a specific database using?", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I can help you find out how much space a specific database is using. However, I need to know which database you'd like to check.\n\nCould you please provide the name of the database you want to analyze?\n\nOnce you give me the database name, I'll use the `dba_databaseSpace` tool to retrieve its disk space allocation and usage information.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_tableUsageImpact_missing_db", - "case_type": "missing_parameter", - "description": "Agent must ask which database to evaluate usage impact for", - "input": "Which users are hitting a particular database the most?", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you identify which users are most actively querying a particular database. However, I need to know which database you want to analyze.\n\n**Could you please specify the database name?**\n\nOnce you provide the database name, I'll use the `dba_tableUsageImpact` tool to show you:\n- Which users are generating the most query activity\n- Which tables are being accessed most frequently\n- Resource consumption patterns (CPU, IO, etc.)\n- Query counts and activity metrics\n\nFor example, if you want to analyze a database called \"SALES_DB\" or \"CUSTOMER_DATA\", just let me know the name and I'll retrieve the usage statistics for you.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "dba_databaseSpace_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which database, then calls dba_databaseSpace after user clarifies", - "input": "Turn 1: How much space is a specific database using? | Turn 2: How much disk space is the demo_user database using?", - "expected_tools": [ - { - "name": "dba_databaseSpace", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_databaseSpace", - "params": { - "database_name": "demo_user" - } - } - ], - "actual_output": "The **demo_user** database has the following disk space usage:\n\n- **Space Allocated**: 78.0 GB\n- **Space Used**: 0.0 GB\n- **Free Space**: 78.0 GB\n- **Percent Used**: 0.0%\n\nThe database currently has 78 GB of space allocated to it but is not using any of that space at the moment. This suggests the database either has no tables/data yet, or the tables are empty.", - "metric_reasons": [], - "recommendation": null, - "turn_details": [ - { - "turn": 1, - "input": "How much space is a specific database using?", - "mode": "clarification", - "passed": true, - "actual_tools": [] - }, - { - "turn": 2, - "input": "How much disk space is the demo_user database using?", - "mode": "tool", - "passed": true, - "actual_tools": [ - { - "name": "dba_databaseSpace", - "params": { - "database_name": "demo_user" - } - } - ] - } - ] - }, - { - "case_id": "dba_tableSqlList_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which table, then calls dba_tableSqlList after user clarifies", - "input": "Turn 1: Show me the queries from the last 14 days | Turn 2: List SQL run against evals_orders over the past 14 days", - "expected_tools": [ - { - "name": "dba_tableSqlList", - "params": { - "table_name": "evals_orders", - "no_days": 14 - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_tableSqlList", - "params": { - "table_name": "evals_orders", - "no_days": 14 - } - } - ], - "actual_output": "Here are the SQL queries that have been run against the **evals_orders** table over the past 14 days:\n\n## Query Summary\n\nAll queries were executed by **DEMO_USER** on **June 23, 2026**. Here are the distinct types of queries found:\n\n### 1. **Data Exploration Queries**\n```sql\nSELECT TOP 5 * FROM demo_user.evals_orders\n```\n- Simple preview queries to see sample data\n\n### 2. **Filtered Queries**\n```sql\nSELECT * FROM demo_user.evals_orders WHERE amount > 500\n```\n- Filtering orders where amount exceeds 500\n\n### 3. **Count Queries**\n```sql\nSELECT COUNT(*) FROM demo_user.evals_orders\n```\n- Getting total row counts\n\n### 4. **Schema Inspection**\n```sql\nSHOW TABLE demo_user.evals_orders\n```\n- Checking table structure and metadata\n\n### 5. **Complex Analytics Query**\n```sql\nLOCKING ROW FOR ACCESS\nSELECT\n TRIM(QTU2.DatabaseName) AS \"DatabaseName\",\n TRIM(QTU2.TableName) AS \"TableName\",\n COUNT(DISTINCT QTU1.QueryID) AS \"QueryCount\",\n (current_timestamp - min(QTU2.CollectTimeStamp)) day(4) AS \"FirstQueryDaysAgo\",\n (current_timestamp - max(QTU2.CollectTimeStamp)) day(4) AS \"LastQueryDaysAgo\"\nFROM (... complex join logic ...)\n```\n- Advanced query analyzing table affinity and usage patterns (likely from the system's own metadata tracking)\n\n## Activity Pattern\n- **Most Recent Activity**: June 23, 2026 at 21:17 (9:17 PM)\n- **Primary User**: DEMO_USER\n- **Query Types**: Mix of exploratory queries, basic filtering, and metadata inspection\n\nThe queries show typical exploratory and analytical activity, with users examining the data, checking counts, and filtering based on the `amount` column.", - "metric_reasons": [], - "recommendation": null, - "turn_details": [ - { - "turn": 1, - "input": "Show me the queries from the last 14 days", - "mode": "clarification", - "passed": true, - "actual_tools": [] - }, - { - "turn": 2, - "input": "List SQL run against evals_orders over the past 14 days", - "mode": "tool", - "passed": true, - "actual_tools": [ - { - "name": "dba_tableSqlList", - "params": { - "table_name": "evals_orders", - "no_days": 14 - } - } - ] - } - ] - }, - { - "case_id": "dba_userSqlList_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which user, then calls dba_userSqlList after user clarifies", - "input": "Turn 1: Show me the queries a user executed over the past 14 days | Turn 2: List all SQL that user ANALYTICS_USER has run over the last 14 days", - "expected_tools": [ - { - "name": "dba_userSqlList", - "params": { - "user_name": "ANALYTICS_USER", - "no_days": 14 - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "dba_userSqlList", - "params": { - "user_name": "ANALYTICS_USER", - "no_days": 14 - } - } - ], - "actual_output": "The query completed successfully, but **no SQL statements were found** for user **ANALYTICS_USER** over the last 14 days.\n\nThis could mean:\n- The user hasn't executed any queries in the past 14 days\n- The user account name might be slightly different (case-sensitive or has extra spaces)\n- Query logging may not be enabled or may have been purged\n- The user might be running queries under a different username\n\n**Would you like me to:**\n1. Check if there are any active sessions for this user?\n2. Verify the exact username exists in the system?\n3. Try a different time period or different username?", - "metric_reasons": [], - "recommendation": null, - "turn_details": [ - { - "turn": 1, - "input": "Show me the queries a user executed over the past 14 days", - "mode": "clarification", - "passed": true, - "actual_tools": [] - }, - { - "turn": 2, - "input": "List all SQL that user ANALYTICS_USER has run over the last 14 days", - "mode": "tool", - "passed": true, - "actual_tools": [ - { - "name": "dba_userSqlList", - "params": { - "user_name": "ANALYTICS_USER", - "no_days": 14 - } - } - ] - } - ] - }, - { - "case_id": "dba_unknown_tool_missing", - "case_type": "missing_parameter", - "description": "User asks for something no DBA tool supports \u2014 agent should explain and ask for clarification", - "input": "Automatically redistribute my table data across AMPs to improve skew", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I understand you want to redistribute table data across AMPs to improve skew, but I need more information to help you effectively:\n\n## Required Information:\n\n1. **Which table** do you want to redistribute? (Please provide the full table name)\n2. **Which database** is it in?\n\n## Additional Context That Would Help:\n\n3. Do you know if the table currently has high skew? (Have you measured CPU skew or data distribution skew?)\n4. What is the current Primary Index (PI) of the table?\n5. Do you have a candidate column or set of columns in mind for a new Primary Index?\n\n## What I Can Do For You:\n\nOnce you provide the table and database name, I can:\n\n- **Analyze current skew patterns** using SQL optimization tools (check CPU skew, I/O skew)\n- **Examine the table's schema** and current Primary Index\n- **Identify high-cardinality columns** that might make good PI candidates\n- **Generate a redistribution strategy** (ALTER TABLE with new UPI or NUPI)\n- **Create the DDL** to redistribute the data\n\n## Note on Redistribution:\n\nRedistributing data typically involves:\n- Changing the Primary Index definition (which controls AMP distribution)\n- This triggers a full table redistribution across all AMPs\n- For large tables, this can be resource-intensive\n\n**Please provide the table name and database name so I can analyze and help you redistribute the data.**", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "plot_line_chart_happy", - "case_type": "happy_path", - "description": "Line chart of order amounts over time from evals_orders", - "input": "Show me a line chart of order amounts over time from demo_user.evals_orders using order_date on the x-axis and amount on the y-axis", - "expected_tools": [ - { - "name": "plot_line_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "order_date", - "columns": [ - "amount" - ] - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "plot_line_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "order_date", - "columns": "amount" - } - }, - { - "name": "base_readQuery", - "params": { - "sql": "SELECT order_date, CAST(amount AS FLOAT) as amount FROM demo_user.evals_orders ORDER BY order_date" - } - } - ], - "actual_output": "Perfect! Here's the data from your evals_orders table showing order amounts over time:\n\n## Order Amounts Over Time - Line Chart Data\n\n| Order Date | Amount |\n|------------|---------|\n| 2024-01-05 | $1,250.00 |\n| 2024-01-12 | -$200.00 |\n| 2024-02-03 | $875.50 |\n| 2024-02-17 | $3,400.00 |\n| 2024-03-01 | $560.00 |\n| 2024-03-15 | -$150.00 |\n| 2024-04-02 | $2,100.00 |\n| 2024-04-20 | $930.00 |\n| 2024-05-08 | $1,780.00 |\n| 2024-05-25 | $4,200.00 |\n\nThe data shows order amounts from January through May 2024. Notable observations:\n- **Highest order**: $4,200 on May 25, 2024\n- **Two negative amounts**: -$200 (Jan 12) and -$150 (Mar 15), which may represent returns or refunds\n- **Overall trend**: Generally positive with some fluctuation, peaking in late May\n\nThe line chart would show the progression of order amounts over this ~5-month period, with the negative values creating dips in January and March, and a strong upward movement toward the May peak.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "plot_pie_chart_happy", - "case_type": "happy_path", - "description": "Pie chart breaking down orders by product category", - "input": "Plot demo_user.evals_orders as a pie chart using product_category for the slice labels and amount for the values.", - "expected_tools": [ - { - "name": "plot_pie_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "product_category", - "column": "amount" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "plot_pie_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "product_category", - "column": "amount" - } - }, - { - "name": "base_readQuery", - "params": { - "sql": "SELECT product_category, CAST(SUM(amount) AS FLOAT) as amount FROM demo_user.evals_orders GROUP BY product_category" - } - } - ], - "actual_output": "I can see the data aggregated by product category. Here's the breakdown:\n\n**Amount by Product Category:**\n- **Electronics**: $6,430.00 (largest category)\n- **Sports**: $6,300.00\n- **Food**: $1,805.50\n- **Books**: $560.00\n- **Clothing**: -$350.00 (negative value, likely returns/refunds)\n\nUnfortunately, there appears to be a technical issue with the direct pie chart plotting function and decimal data types. The data shows that Electronics and Sports dominate the sales, together accounting for the vast majority of the total amount, while Clothing has a negative value which may represent returns or refunds.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "plot_polar_chart_happy", - "case_type": "happy_path", - "description": "Polar area chart of order amounts by product category", - "input": "Create a polar area chart showing order amounts by product_category from demo_user.evals_orders", - "expected_tools": [ - { - "name": "plot_polar_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "product_category", - "column": "amount" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "plot_polar_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "product_category", - "column": "order_amount" - } - }, - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - }, - { - "name": "plot_polar_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "product_category", - "column": "amount" - } - }, - { - "name": "base_readQuery", - "params": { - "sql": "SELECT product_category, CAST(SUM(amount) AS FLOAT) as total_amount FROM demo_user.evals_orders GROUP BY product_category", - "persist": true - } - }, - { - "name": "plot_polar_chart", - "params": { - "column": "total_amount", - "labels": "product_category", - "table_name": "vt_d842dc42_33bd_41" - } - } - ], - "actual_output": "", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "plot_radar_chart_happy", - "case_type": "happy_path", - "description": "Radar chart of salary by department in evals_employees", - "input": "Draw a spider chart from demo_user.evals_employees using department as the axis labels and salary as the values.", - "expected_tools": [ - { - "name": "plot_radar_chart", - "params": { - "table_name": "demo_user.evals_employees", - "labels": "department", - "columns": "salary" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "plot_radar_chart", - "params": { - "table_name": "demo_user.evals_employees", - "labels": "department", - "columns": "salary" - } - }, - { - "name": "base_readQuery", - "params": { - "sql": "SELECT department, CAST(AVG(salary) AS DECIMAL(18,2)) as avg_salary \nFROM demo_user.evals_employees \nGROUP BY department \nORDER BY department" - } - } - ], - "actual_output": "I see the data shows average salaries by department. Unfortunately, there appears to be a technical issue with the radar chart function handling the Decimal data type from the database. \n\nHere's the salary data by department that would be displayed in the spider chart:\n\n**Department Salary Overview:**\n- **Engineering**: $95,000 (highest)\n- **Finance**: $88,000\n- **Marketing**: $77,000\n- **Sales**: $70,750\n- **HR**: $68,000 (lowest)\n\nThe spider/radar chart would show these five departments as points around the perimeter, with salary values determining how far from the center each point extends. Engineering would have the longest spoke, followed by Finance, Marketing, Sales, and HR.\n\nThis visualization would help compare salary levels across departments at a glance, showing that Engineering has the highest average salaries while HR has the lowest among the five departments.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "plot_line_vs_radar_ambiguous", - "case_type": "ambiguous_selection", - "description": "Time-series data should use line chart, not radar", - "input": "Plot how order amounts change over time from demo_user.evals_orders using order_date on the x-axis", - "expected_tools": [ - { - "name": "plot_line_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "order_date", - "columns": [ - "amount" - ] - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "plot_line_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "order_date", - "columns": "order_amount" - } - }, - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - }, - { - "name": "plot_line_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "order_date", - "columns": "amount" - } - }, - { - "name": "base_readQuery", - "params": { - "sql": "SELECT order_date, CAST(amount AS FLOAT) as amount FROM demo_user.evals_orders ORDER BY order_date" - } - } - ], - "actual_output": "Perfect! I can see the data now. Here's a summary of how order amounts change over time from January to May 2024:\n\n**Order Amounts Over Time (demo_user.evals_orders)**\n\n| Order Date | Amount |\n|------------|---------|\n| 2024-01-05 | $1,250.00 |\n| 2024-01-12 | -$200.00 (return/refund) |\n| 2024-02-03 | $875.50 |\n| 2024-02-17 | $3,400.00 |\n| 2024-03-01 | $560.00 |\n| 2024-03-15 | -$150.00 (return/refund) |\n| 2024-04-02 | $2,100.00 |\n| 2024-04-20 | $930.00 |\n| 2024-05-08 | $1,780.00 |\n| 2024-05-25 | $4,200.00 |\n\n**Key Observations:**\n- The data spans from January to May 2024 with 10 orders\n- Order amounts range from -$200 to $4,200\n- There are two negative amounts (returns/refunds) in January and March\n- The highest order amount ($4,200) occurred on May 25th\n- February 17th also had a significant order at $3,400\n\nThe trend shows some volatility with occasional high-value orders and a couple of returns, with the highest activity value occurring at the end of May.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "plot_pie_vs_polar_ambiguous", - "case_type": "ambiguous_selection", - "description": "Proportion/share breakdown should use pie chart over polar", - "input": "Use a pie chart to show demo_user.evals_orders with product_category as labels and amount as the values.", - "expected_tools": [ - { - "name": "plot_pie_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "product_category", - "column": "amount" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "plot_pie_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "product_category", - "column": "amount" - } - }, - { - "name": "base_readQuery", - "params": { - "sql": "SELECT product_category, SUM(amount) as total_amount FROM demo_user.evals_orders GROUP BY product_category" - } - } - ], - "actual_output": "Based on the data from demo_user.evals_orders, here's the breakdown by product category:\n\n## Product Category Distribution by Amount\n\n| Product Category | Total Amount |\n|-----------------|--------------|\n| **Electronics** | $6,430.00 |\n| **Sports** | $6,300.00 |\n| **Food** | $1,805.50 |\n| **Books** | $560.00 |\n| **Clothing** | -$350.00 |\n\n**Total:** $14,745.50\n\n**Key Insights:**\n- **Electronics** and **Sports** are the top categories, each accounting for over $6,000 in sales\n- **Food** represents a moderate amount at $1,805.50\n- **Books** contributes $560.00\n- **Clothing** shows a negative value (-$350.00), which may indicate returns or refunds\n\nNote: The pie chart visualization encountered a technical issue with decimal number formatting, but the data summary above provides the complete picture of how amounts are distributed across product categories.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "plot_missing_data", - "case_type": "missing_parameter", - "description": "Chart request with no table or data provided \u2014 agent must ask for clarification", - "input": "Create a chart for me", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to create a chart for you! To do that, I need a bit more information:\n\n1. **What type of chart** would you like?\n - Line chart (for trends over time)\n - Pie chart (for proportions/breakdowns)\n - Polar chart (for circular category comparisons)\n - Radar/spider chart (for multi-dimensional comparisons)\n\n2. **What data** should be visualized?\n - Which table/database contains the data?\n - What columns should be displayed?\n - Any specific filtering or time period?\n\nFor example, you could say:\n- \"Create a line chart showing sales over time from the Sales table\"\n- \"Create a pie chart of revenue by region from the Revenue_Data table\"\n- \"Show me a radar chart comparing product metrics\"\n\nWhat would you like to visualize?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "plot_pie_chart_direct", - "case_type": "happy_path", - "description": "Direct pie chart from table \u2014 no pre-query needed since plot tools read tables directly", - "input": "Give me a pie chart of amount by product_category from demo_user.evals_orders", - "expected_tools": [ - { - "name": "plot_pie_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "product_category", - "column": "amount" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "plot_pie_chart", - "params": { - "table_name": "demo_user.evals_orders", - "labels": "product_category", - "column": "amount" - } - }, - { - "name": "base_readQuery", - "params": { - "sql": "SELECT product_category, SUM(amount) as total_amount FROM demo_user.evals_orders GROUP BY product_category" - } - } - ], - "actual_output": "Here's the pie chart breakdown of amount by product category from demo_user.evals_orders:\n\n**Amount by Product Category:**\n\n| Product Category | Total Amount |\n|-----------------|--------------|\n| **Electronics** | $6,430.00 (44.6%) |\n| **Sports** | $6,300.00 (43.7%) |\n| **Food** | $1,805.50 (12.5%) |\n| **Books** | $560.00 (3.9%) |\n| **Clothing** | -$350.00 (-2.4%) |\n\n**Key Insights:**\n- **Electronics** and **Sports** dominate, together accounting for 88.3% of total amount\n- **Clothing** shows a negative amount (-$350), which may indicate returns or refunds\n- Total amount across all categories: **$14,745.50**\n\nNote: The negative value in Clothing suggests there were returns or credit transactions that exceeded sales in that category.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "plot_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask which table to plot", - "input": "Create a line chart of sales over time", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you create a line chart of sales over time! However, I need a few more details to get started:\n\n1. **Table name**: Which table contains the sales data?\n2. **Database name**: Which database is this table in?\n3. **Date/Time column**: What is the name of the column that contains the date or time information?\n4. **Sales column**: What is the name of the column that contains the sales values?\n\nFor example, if you have a table called `sales_data` in database `analytics` with columns `sale_date` and `sales_amount`, I would use those.\n\nCould you provide these details so I can create the line chart for you?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "plot_unknown_tool_missing", - "case_type": "missing_parameter", - "description": "User asks for a non\u2011existent plot tool", - "input": "Show me the network topology as a diagram", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I understand you'd like to see a network topology diagram. However, I need more specific information to help you with this request:\n\n1. **What type of network topology are you referring to?**\n - Database dependency graph (tables, views, and their relationships)?\n - Data lineage network (upstream/downstream dependencies)?\n - System architecture diagram?\n\n2. **Which database or objects would you like to analyze?**\n - A specific database name?\n - Specific tables or views?\n - A pattern like '%WBC%' or '%SALES%'?\n\nThe available tools I have can help you visualize:\n\n- **Database dependency graphs**: Using graph analysis tools, I can show you:\n - Root objects (starting points with no dependencies)\n - Downstream dependencies (what depends on what)\n - Upstream dependencies (what objects depend on)\n - Circular dependencies (cycles)\n - Connected components (isolated groups)\n\n- **Table relationship networks**: Show which tables are commonly queried together (affinity analysis)\n\nCould you please clarify:\n- What database or schema you want to analyze?\n- What specific aspect of the topology you're interested in (dependencies, lineage, relationships)?\n\nOnce you provide this information, I can generate the appropriate network diagram for you.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_missingValues_happy", - "case_type": "happy_path", - "description": "Find columns with nulls in evals_orders (ship_date is nullable)", - "input": "Which columns in demo_user.evals_orders contain null or empty entries?", - "expected_tools": [ - { - "name": "qlty_missingValues", - "params": {} - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_missingValues", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "actual_output": "Based on the analysis of the `demo_user.evals_orders` table, here are the columns with NULL or empty entries:\n\n## Columns with Missing Values:\n\n**1. ship_date**\n- **NULL Count:** 3\n- **NULL Percentage:** 30.0%\n\nThis is the only column that contains NULL values in the table.\n\n## Columns with No Missing Values:\n\nThe following columns have no NULL or missing values:\n- `order_id`\n- `customer_name`\n- `order_date`\n- `product_category`\n- `amount`\n- `quantity`\n\nThe `ship_date` column has 3 NULL entries out of 10 total rows (30%), which could indicate orders that haven't been shipped yet or missing shipping information.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_missingValues_ambiguous_vs_rowsWithMissingValues", - "case_type": "ambiguous_selection", - "description": "Asking which columns have nulls (column-level summary) should prefer qlty_missingValues over qlty_rowsWithMissingValues", - "input": "Which fields in demo_user.evals_orders have missing data?", - "expected_tools": [ - { - "name": "qlty_missingValues", - "params": {} - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_missingValues", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "actual_output": "The `demo_user.evals_orders` table has missing data in **one field**:\n\n- **ship_date**: 3 null values (30.0% of records)\n\nAll other fields have complete data with no missing values:\n- order_id\n- quantity\n- amount\n- customer_name\n- order_date\n- product_category\n\nSo about 30% of the orders in this table are missing their shipping dates, which might indicate orders that haven't been shipped yet or incomplete data entry.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_negativeValues_happy", - "case_type": "happy_path", - "description": "Find negative amounts in evals_orders (some amounts are negative refunds)", - "input": "Show me which fields in demo_user.evals_orders contain numbers below zero", - "expected_tools": [ - { - "name": "qlty_negativeValues", - "params": {} - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_negativeValues", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "actual_output": "Based on the analysis of the `demo_user.evals_orders` table, here are the fields that contain numbers below zero:\n\n**Field with Negative Values:**\n- **amount** - Contains 2 records with negative values\n\n**Fields with No Negative Values:**\n- **quantity** - 0 negative values\n- **order_id** - 0 negative values\n\nThe other fields (ship_date, product_category, order_date, customer_name) are non-numeric fields, so negative value checking doesn't apply to them.\n\nThe `amount` field is the only column in your table that has negative numbers, with 2 instances of values below zero.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_distinctCategories_happy", - "case_type": "happy_path", - "description": "List unique product categories in evals_orders", - "input": "Show me all the unique values in the product_category column of demo_user.evals_orders", - "expected_tools": [ - { - "name": "qlty_distinctCategories", - "params": { - "column_name": "product_category" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_distinctCategories", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders", - "column_name": "product_category" - } - } - ], - "actual_output": "Perfect! Here are all the unique values in the **product_category** column of **demo_user.evals_orders**:\n\n| Product Category | Count of Records |\n|-----------------|------------------|\n| **Electronics** | 3 |\n| **Books** | 1 |\n| **Clothing** | 2 |\n| **Sports** | 2 |\n| **Food** | 2 |\n\nThere are **5 distinct product categories** in total, with Electronics being the most common (appearing 3 times) and Books being the least common (appearing once).", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_distinctCategories_missing_column", - "case_type": "missing_parameter", - "description": "Agent must ask which column when none is specified", - "input": "What distinct values exist in a column of demo_user.evals_orders?", - "expected_tools": [], - "passed": false, - "failure_stage": "deterministic", - "failure_detail": "expected no tool calls for missing_parameter case, got ['base_columnDescription']", - "actual_tools": [ - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "actual_output": "The `demo_user.evals_orders` table has the following columns:\n\n1. **product_category** (VARCHAR)\n2. **ship_date** (DATE)\n3. **customer_name** (VARCHAR)\n4. **order_id** (INTEGER)\n5. **quantity** (INTEGER)\n6. **amount** (DECIMAL)\n7. **order_date** (DATE)\n\nWhich column would you like to see the distinct values for?", - "metric_reasons": [], - "recommendation": "The prompt deliberately omits required information: \"What distinct values exist in a column of demo_user.evals_orders?\". The agent called `base_columnDescription` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response.", - "turn_details": null - }, - { - "case_id": "qlty_standardDeviation_happy", - "case_type": "happy_path", - "description": "Get mean and spread of salary in evals_employees", - "input": "What is the average and spread of salary values in demo_user.evals_employees?", - "expected_tools": [ - { - "name": "qlty_standardDeviation", - "params": { - "column_name": "salary" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_standardDeviation", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees", - "column_name": "salary" - } - } - ], - "actual_output": "Based on the analysis of the salary column in demo_user.evals_employees:\n\n**Average Salary:** $83,055.56\n\n**Standard Deviation (Spread):** $12,053.12\n\nThis means:\n- The typical salary in this dataset is around $83,056\n- Most salaries fall within approximately one standard deviation of the mean, which would be roughly between $71,002 and $95,109\n- The standard deviation of about $12,053 indicates moderate variability in salary values across the employees in this table", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_standardDeviation_ambiguous_vs_univariateStatistics", - "case_type": "ambiguous_selection", - "description": "Asking specifically for standard deviation should prefer qlty_standardDeviation over the broader qlty_univariateStatistics", - "input": "What is the standard deviation of the salary field in demo_user.evals_employees?", - "expected_tools": [ - { - "name": "qlty_standardDeviation", - "params": { - "column_name": "salary" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_standardDeviation", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees", - "column_name": "salary" - } - } - ], - "actual_output": "The standard deviation of the salary field in demo_user.evals_employees is **12,053.12**.\n\nAdditionally, the mean (average) salary is **83,055.56**.\n\nThis indicates that salaries in this table vary by approximately $12,053 from the average, which suggests a moderate spread in the salary distribution.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_columnSummary_happy", - "case_type": "happy_path", - "description": "Get a statistical summary of all columns in evals_employees", - "input": "Show me a statistical summary of every field in demo_user.evals_employees", - "expected_tools": [ - { - "name": "qlty_columnSummary", - "params": {} - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_columnSummary", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "actual_output": "Here's a statistical summary of every field in **demo_user.evals_employees**:\n\n## String/Character Fields\n\n| Column | Data Type | Non-Null Count | Null Count | Blank Count | Null % |\n|--------|-----------|----------------|------------|-------------|--------|\n| **name** | VARCHAR(100) | 10 | 0 | 0 | 0.0% |\n| **region** | VARCHAR(50) | 10 | 0 | 0 | 0.0% |\n| **department** | VARCHAR(50) | 10 | 0 | 0 | 0.0% |\n\n## Numeric Fields\n\n| Column | Data Type | Non-Null Count | Null Count | Zero Count | Positive Count | Negative Count | Null % |\n|--------|-----------|----------------|------------|------------|----------------|----------------|--------|\n| **employee_id** | INTEGER | 10 | 0 | 0 | 10 | 0 | 0.0% |\n| **manager_id** | INTEGER | 9 | 1 | 0 | 9 | 0 | 10.0% |\n| **salary** | DECIMAL(10,2) | 9 | 1 | 0 | 9 | 0 | 10.0% |\n\n## Date Fields\n\n| Column | Data Type | Non-Null Count | Null Count | Null % |\n|--------|-----------|----------------|------------|--------|\n| **hire_date** | DATE | 10 | 0 | 0.0% |\n\n### Key Findings:\n- **Total rows**: 10 employees\n- **Complete fields**: name, region, department, employee_id, and hire_date have 100% data completeness\n- **Missing data**: manager_id and salary each have 1 missing value (10% null rate)\n- **All numeric values are positive**: No negative or zero values in employee_id, manager_id, or salary\n- **No blank strings**: All text fields are properly populated", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_univariateStatistics_happy", - "case_type": "happy_path", - "description": "Get full univariate stats for salary in evals_employees", - "input": "Give me a complete statistical breakdown of the salary field in demo_user.evals_employees", - "expected_tools": [ - { - "name": "qlty_univariateStatistics", - "params": { - "column_name": "salary" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_univariateStatistics", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees", - "column_name": "salary" - } - } - ], - "actual_output": "## Complete Statistical Breakdown - Salary Field (demo_user.evals_employees)\n\n### **Central Tendency Measures**\n- **Mean**: $83,055.56\n- **Median**: $85,000.00\n- **Mode**: $68,000.00\n- **Geometric Mean**: $82,278.64\n- **Harmonic Mean**: $81,508.55\n- **Trimmed Mean**: $82,500.00\n\n### **Dispersion Measures**\n- **Standard Deviation**: $12,053.12\n- **Variance**: $145,277,777.78\n- **Coefficient of Variation**: 14.51%\n- **Range**: $34,000.00\n- **Interquartile Range**: $19,000.00\n- **Standard Error**: $4,017.71\n\n### **Range & Extremes**\n- **Minimum**: $68,000.00\n- **Maximum**: $102,000.00\n- **Bottom 5 Values**: $68,000, $69,500, $72,000, $77,000, $85,000\n- **Top 5 Values**: $102,000, $95,000, $91,000, $88,000, $85,000\n\n### **Percentiles**\n- **1st Percentile**: $68,120.00\n- **5th Percentile**: $68,600.00\n- **10th Percentile**: $69,200.00\n- **25th Percentile (Q1)**: $72,000.00\n- **50th Percentile (Q2/Median)**: $85,000.00\n- **75th Percentile (Q3)**: $91,000.00\n- **90th Percentile**: $96,400.00\n- **95th Percentile**: $99,200.00\n- **99th Percentile**: $101,440.00\n\n### **Distribution Shape**\n- **Skewness**: 0.152 (slightly right-skewed, nearly symmetric)\n- **Kurtosis**: -1.333 (platykurtic - flatter than normal distribution)\n\n### **Data Quality**\n- **Count**: 9 salary values\n- **Null Count**: 1 missing value\n- **Unique Values**: 9\n- **Zero Values**: 0\n- **Negative Values**: 0\n- **Positive Values**: 9\n\n### **Aggregate Measures**\n- **Sum**: $747,500.00\n- **Corrected Sum of Squares**: $1,162,222,222.22\n- **Uncorrected Sum of Squares**: $63,246,250,000.00\n\n### **Key Insights**\n\u2713 The salary distribution is fairly balanced with a mean of $83,055 and median of $85,000 \n\u2713 Low coefficient of variation (14.51%) indicates moderate consistency \n\u2713 Slight positive skew suggests a few higher salaries pulling the mean slightly up \n\u2713 The interquartile range of $19,000 shows reasonable mid-range spread \n\u2713 One employee has a missing salary value", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_univariateStatistics_ambiguous_vs_columnSummary", - "case_type": "ambiguous_selection", - "description": "Asking for stats on a specific column should prefer qlty_univariateStatistics over the table-level qlty_columnSummary", - "input": "Show me min, max, mean, and percentiles for the amount column in demo_user.evals_orders", - "expected_tools": [ - { - "name": "qlty_univariateStatistics", - "params": { - "column_name": "amount" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_univariateStatistics", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders", - "column_name": "amount" - } - } - ], - "actual_output": "Here are the key statistics for the **amount** column in **demo_user.evals_orders**:\n\n## Summary Statistics\n\n- **Minimum**: -200.0\n- **Maximum**: 4,200.0\n- **Mean**: 1,474.55\n- **Median**: 1,090.0\n\n## Percentiles\n\n| Percentile | Value |\n|------------|--------|\n| 1st | -195.5 |\n| 5th | -177.5 |\n| 10th | -155.0 |\n| 25th (Q1) | 638.9 |\n| 50th (Median) | 1,090.0 |\n| 75th (Q3) | 2,020.0 |\n| 90th | 3,480.0 |\n| 95th | 3,840.0 |\n| 99th | 4,128.0 |\n\n## Additional Insights\n\n- **Standard Deviation**: 1,437.8\n- **Count**: 10 records\n- **Range**: 4,400.0 (from -200 to 4,200)\n- **Negative Values**: 2 records have negative amounts\n- **Interquartile Range (IQR)**: 1,381.1\n\nThe data shows a right-skewed distribution (skewness = 0.82) with some negative values that appear to be returns or refunds, and most values clustering in the lower-to-mid range with a few high-value outliers.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_rowsWithMissingValues_happy", - "case_type": "happy_path", - "description": "Retrieve rows where ship_date is null in evals_orders", - "input": "Show me all records from demo_user.evals_orders where the ship_date is empty or null", - "expected_tools": [ - { - "name": "qlty_rowsWithMissingValues", - "params": { - "column_name": "ship_date" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_rowsWithMissingValues", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders", - "column_name": "ship_date" - } - } - ], - "actual_output": "I found **3 records** from demo_user.evals_orders where the ship_date is null:\n\n| Order ID | Customer Name | Order Date | Ship Date | Amount | Product Category | Quantity |\n|----------|---------------|------------|-----------|---------|------------------|----------|\n| 1009 | Iota Inc | 2024-05-08 | null | 1780.0 | Electronics | 1 |\n| 1006 | Zeta Partners | 2024-03-15 | null | -150.0 | Clothing | 1 |\n| 1003 | Gamma Inc | 2024-02-03 | null | 875.5 | Food | 5 |\n\nAll three of these orders have no ship date recorded, indicating they may not have been shipped yet or the shipping information wasn't entered into the system.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_rowsWithMissingValues_ambiguous_vs_missingValues", - "case_type": "ambiguous_selection", - "description": "Asking to retrieve the actual rows with missing data should prefer qlty_rowsWithMissingValues over qlty_missingValues", - "input": "Fetch the rows from demo_user.evals_orders that are missing a ship_date", - "expected_tools": [ - { - "name": "qlty_rowsWithMissingValues", - "params": { - "column_name": "ship_date" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_rowsWithMissingValues", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders", - "column_name": "ship_date" - } - } - ], - "actual_output": "I found **3 rows** in demo_user.evals_orders that are missing a ship_date:\n\n| order_id | customer_name | order_date | ship_date | amount | product_category | quantity |\n|----------|---------------|------------|-----------|---------|------------------|----------|\n| 1009 | Iota Inc | 2024-05-08 | NULL | 1780.0 | Electronics | 1 |\n| 1006 | Zeta Partners | 2024-03-15 | NULL | -150.0 | Clothing | 1 |\n| 1003 | Gamma Inc | 2024-02-03 | NULL | 875.5 | Food | 5 |\n\nThese are orders that have been placed but not yet shipped.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_multi_tool_summary_then_nulls", - "case_type": "multi_tool", - "description": "Get column summary then drill into rows with missing values", - "input": "First give me a column summary of demo_user.evals_orders, then show me the rows where ship_date is missing", - "expected_tools": [ - { - "name": "qlty_columnSummary", - "params": {} - }, - { - "name": "qlty_rowsWithMissingValues", - "params": { - "column_name": "ship_date" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "qlty_columnSummary", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - }, - { - "name": "qlty_rowsWithMissingValues", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders", - "column_name": "ship_date" - } - } - ], - "actual_output": "## Column Summary for demo_user.evals_orders\n\nThe table has **7 columns** with the following characteristics:\n\n| Column Name | Data Type | Non-Null Count | Null Count | Null % | Additional Info |\n|-------------|-----------|----------------|------------|--------|-----------------|\n| **order_id** | INTEGER | 10 | 0 | 0% | All positive values |\n| **customer_name** | VARCHAR(100) | 10 | 0 | 0% | No blanks |\n| **order_date** | DATE | 10 | 0 | 0% | Complete |\n| **ship_date** | DATE | 7 | **3** | **30%** | Missing for 3 orders |\n| **amount** | DECIMAL(10,2) | 10 | 0 | 0% | 8 positive, 2 negative |\n| **product_category** | VARCHAR(50) | 10 | 0 | 0% | No blanks |\n| **quantity** | INTEGER | 10 | 0 | 0% | All positive values |\n\n---\n\n## Rows Where ship_date is Missing\n\nThere are **3 orders** with missing ship dates:\n\n| Order ID | Customer Name | Order Date | Ship Date | Amount | Product Category | Quantity |\n|----------|---------------|------------|-----------|--------|------------------|----------|\n| **1009** | Iota Inc | 2024-05-08 | NULL | $1,780.00 | Electronics | 1 |\n| **1006** | Zeta Partners | 2024-03-15 | NULL | -$150.00 | Clothing | 1 |\n| **1003** | Gamma Inc | 2024-02-03 | NULL | $875.50 | Food | 5 |\n\n**Note:** Order 1006 (Zeta Partners) has a negative amount of -$150.00, which may indicate a return or credit.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "qlty_unknown_tool_missing", - "case_type": "missing_parameter", - "description": "User asks for a non\u2011existent quality tool", - "input": "Give me a data quality score for the dataset", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you calculate a data quality score for a dataset. However, I need you to provide some information first:\n\n1. **Database name** (if applicable, otherwise I can use the default)\n2. **Table name** - Which specific table/dataset do you want to analyze?\n\nOnce you provide the table name, I can run several data quality checks including:\n- **Missing values analysis** - identifying columns with NULL values\n- **Column summary statistics** - providing an overview of all columns\n- **Negative values check** - for numeric columns where negatives might be anomalous\n- **Univariate statistics** - detailed statistical profiles for numeric columns\n\nPlease share the table name (and database name if you know it), and I'll generate a comprehensive data quality assessment for you.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_userDbPermissions_ambiguous_vs_userRoles", - "case_type": "ambiguous_selection", - "description": "Asking about a user's access rights on databases should prefer sec_userDbPermissions, not sec_userRoles", - "input": "Which databases can the user 'bob' access and what can they do in each one?", - "expected_tools": [ - { - "name": "sec_userDbPermissions", - "params": { - "user_name": "bob" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_userDbPermissions", - "params": { - "user_name": "bob" - } - }, - { - "name": "sec_userRoles", - "params": { - "user_name": "bob" - } - } - ], - "actual_output": "Based on the results:\n\n## Summary for User 'bob':\n\n**Direct Database Permissions:** None found\n\n**Role Memberships:** None found\n\nThis indicates that user 'bob' either:\n1. Does not exist in the system\n2. Has no permissions or roles currently assigned\n3. May only have permissions on their own user database (if one exists with the same name)\n\nIn Teradata, users typically get access to databases through either:\n- **Direct grants** - specific permissions granted directly to the user account\n- **Role memberships** - permissions inherited from roles assigned to the user\n\nSince both queries returned no results, user 'bob' appears to have no database access permissions configured at this time. If 'bob' needs access to databases, a database administrator would need to either:\n- Grant direct permissions to 'bob' on specific databases/tables\n- Assign 'bob' to one or more roles that have the necessary permissions\n\nWould you like me to check if the user 'bob' exists in the system, or would you like to grant them specific permissions?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_userDbPermissions_missing_user", - "case_type": "missing_parameter", - "description": "User asks about permissions without naming a user", - "input": "Show me the database permissions", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you view database permissions! To show you the relevant permissions, I need a bit more information:\n\n**What type of permissions are you looking for?**\n\n1. **User's database permissions** - What databases and tables a specific user can access (SELECT, INSERT, UPDATE, DELETE, etc.)\n - Need: Username\n\n2. **Role's database permissions** - What access rights are granted to a specific role\n - Need: Role name\n\n3. **User's role assignments** - What roles are assigned to a specific user\n - Need: Username\n\nPlease let me know which type you'd like to see and provide the username or role name accordingly.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_rolePermissions_ambiguous_vs_userDbPermissions", - "case_type": "ambiguous_selection", - "description": "Asking about a role's access should use sec_rolePermissions, not sec_userDbPermissions", - "input": "What access rights does the 'analyst' role have across all databases?", - "expected_tools": [ - { - "name": "sec_rolePermissions", - "params": { - "role_name": "analyst" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_rolePermissions", - "params": { - "role_name": "analyst" - } - } - ], - "actual_output": "The query returned no results, which means the 'analyst' role either:\n\n1. **Does not exist** in the system\n2. **Has no database-level permissions granted** to it\n3. **Has no members** (the query joins role members with role rights)\n\nThis could happen if:\n- The role name is different (case-sensitive or spelled differently)\n- The role exists but hasn't been granted any database access rights yet\n- The role has no users assigned to it\n\nWould you like me to:\n- Check if the role exists with a different spelling or case?\n- Look up what roles are available in the system?\n- Check permissions for a specific user instead?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_userRoles_ambiguous_vs_rolePermissions", - "case_type": "ambiguous_selection", - "description": "Listing roles assigned to a user should prefer sec_userRoles, not sec_rolePermissions", - "input": "Which roles are currently assigned to user 'diana'?", - "expected_tools": [ - { - "name": "sec_userRoles", - "params": { - "user_name": "diana" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_userRoles", - "params": { - "user_name": "diana" - } - } - ], - "actual_output": "The user **'diana'** currently has **no roles assigned**.\n\nThe query checked the Teradata system catalogs for any role memberships associated with this user account, but no role assignments were found. This means 'diana' either:\n- Has no roles granted to them\n- Relies on direct database-level permissions instead of role-based access\n- May be a newly created user account\n\nIf you'd like to check what direct database permissions 'diana' has instead, I can run that query using the `sec_userDbPermissions` tool.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_rolePermissions_ambiguous_vs_userRoles", - "case_type": "ambiguous_selection", - "description": "Permissions granted to a named role should prefer sec_rolePermissions, not sec_userRoles", - "input": "What database access has the 'analyst' role been granted?", - "expected_tools": [ - { - "name": "sec_rolePermissions", - "params": { - "role_name": "analyst" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_rolePermissions", - "params": { - "role_name": "analyst" - } - } - ], - "actual_output": "The 'analyst' role currently has **no database access permissions** granted to it. The query returned zero results, which means:\n\n- No direct database-level permissions (SELECT, INSERT, UPDATE, DELETE, etc.) have been granted to this role\n- No CREATE, DROP, or ALTER privileges on database objects\n- No administrative or system-level privileges\n\nThis could mean:\n1. The role exists but hasn't been granted any permissions yet\n2. The role doesn't exist in the system\n3. Access is granted through other means (e.g., individual user permissions rather than role-based permissions)\n\nIf you expected this role to have permissions, you may want to:\n- Check if the role name is spelled correctly\n- Verify if permissions are granted directly to users instead of through this role\n- Check if there are other roles that users should be assigned to instead", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_userRoles_ambiguous_vs_userDbPermissions", - "case_type": "ambiguous_selection", - "description": "Asking about a user's role assignments should prefer sec_userRoles even though permissions is related", - "input": "Tell me which roles have been assigned to user 'diana'", - "expected_tools": [ - { - "name": "sec_userRoles", - "params": { - "user_name": "diana" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_userRoles", - "params": { - "user_name": "diana" - } - } - ], - "actual_output": "The query returned no results for user 'diana'. This means that either:\n\n1. **The user 'diana' does not exist** in the Teradata system, or\n2. **The user 'diana' has no roles assigned** to them\n\nUsers in Teradata can exist without having any roles assigned - they might only have direct database permissions instead. If you'd like, I can check:\n- User 'diana's direct database permissions using `sec_userDbPermissions`\n- Whether the user exists at all by checking session or user information\n\nWould you like me to investigate further?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_multi_tool_user_roles_then_role_permissions", - "case_type": "multi_tool", - "description": "Find a user's roles, then inspect a specific named role's permissions in one pass", - "input": "Show me what roles 'alice' has, then list the permissions for the 'analyst' role", - "expected_tools": [ - { - "name": "sec_userRoles", - "params": { - "user_name": "alice" - } - }, - { - "name": "sec_rolePermissions", - "params": { - "role_name": "analyst" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_userRoles", - "params": { - "user_name": "alice" - } - }, - { - "name": "sec_rolePermissions", - "params": { - "role_name": "analyst" - } - } - ], - "actual_output": "Based on the query results:\n\n## Roles for user 'alice'\n**No roles found** - The user 'alice' does not have any roles assigned in the system.\n\n## Permissions for role 'analyst'\n**No permissions found** - The role 'analyst' either doesn't exist in the system or has no database-level permissions granted to it.\n\n**Possible reasons:**\n1. The user 'alice' may not exist in the system\n2. The role 'analyst' may not exist in the system\n3. User and role names may be case-sensitive - you might want to try different capitalizations (e.g., 'Alice', 'ALICE', 'Analyst', 'ANALYST')\n4. The user exists but simply has no roles assigned\n5. The role exists but has no permissions granted yet\n\nWould you like me to check with different name variations, or would you like to see a list of all available users or roles in the system?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_userDbPermissions_happy", - "case_type": "happy_path", - "description": "Show what databases a user can access and what they can do in each", - "input": "Show me what access rights jsmith has across all the different databases in the system.", - "expected_tools": [ - { - "name": "sec_userDbPermissions", - "params": { - "user_name": "jsmith" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_userDbPermissions", - "params": { - "user_name": "jsmith" - } - } - ], - "actual_output": "The query returned no results for user 'jsmith'. This could mean one of the following:\n\n1. **The user doesn't exist** in the system\n2. **The user has no direct database permissions** granted (they may only have access through role memberships)\n3. **The username might be different** (case-sensitive or different format)\n\nWould you like me to:\n- **Check what roles are assigned to jsmith** (using `sec_userRoles`)? Users often receive their permissions through role memberships rather than direct grants.\n- **Search for similar usernames** if the exact username might be different?\n\nLet me know how you'd like to proceed!", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_rolePermissions_happy", - "case_type": "happy_path", - "description": "Show what permissions a named role has", - "input": "What access rights are granted to the FinanceReporter role?", - "expected_tools": [ - { - "name": "sec_rolePermissions", - "params": { - "role_name": "FinanceReporter" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_rolePermissions", - "params": { - "role_name": "FinanceReporter" - } - } - ], - "actual_output": "The query returned no results for the **FinanceReporter** role. This could mean one of the following:\n\n1. **The role does not exist** in the Teradata system\n2. **The role has no database-level permissions granted** to it\n3. **The role name might be different** (case-sensitive or spelled differently)\n\nTo help troubleshoot, you might want to:\n- Verify the exact role name exists in the system\n- Check if the role has been granted any permissions at all\n- Ensure you have the necessary privileges to view role permissions\n\nWould you like me to help verify if this role exists or check permissions for a different role name?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_userRoles_happy", - "case_type": "happy_path", - "description": "List the roles assigned to a user account", - "input": "Which roles are assigned to the database account jsmith?", - "expected_tools": [ - { - "name": "sec_userRoles", - "params": { - "user_name": "jsmith" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_userRoles", - "params": { - "user_name": "jsmith" - } - } - ], - "actual_output": "The database account **jsmith** currently has **no roles** assigned to it. The query returned zero results, indicating that this user account either:\n\n1. Has no roles explicitly granted to it\n2. Only has direct database permissions (not role-based access)\n\nIf you need to see what direct database permissions jsmith has instead, I can check that using a different query. Would you like me to look up the direct database permissions for jsmith?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_rolePermissions_missing_role", - "case_type": "missing_parameter", - "description": "Agent must ask which role to inspect", - "input": "What access rights does the role have?", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you check the access rights for a role, but I need to know which specific role you'd like me to look up.\n\nCould you please provide the name of the Teradata role you want to investigate?", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_userRoles_missing_user", - "case_type": "missing_parameter", - "description": "Agent must ask which user to retrieve roles for", - "input": "Show me the user roles", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'll retrieve the list of user roles from the system. However, I need to clarify: would you like to see:\n\n1. **All roles that exist in the system** (role definitions)\n2. **Roles assigned to a specific user** (requires a username)\n3. **Permissions granted to a specific role** (requires a role name)\n\nCould you please clarify which information you're looking for? If you want to see roles for a specific user or details about a specific role, please provide the username or role name.", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - }, - { - "case_id": "sec_userRoles_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which user, then calls sec_userRoles after user clarifies", - "input": "Turn 1: Show me the user roles | Turn 2: Which roles are assigned to user diana?", - "expected_tools": [ - { - "name": "sec_userRoles", - "params": { - "user_name": "diana" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_userRoles", - "params": { - "user_name": "diana" - } - } - ], - "actual_output": "The query returned no results, which means **user 'diana' currently has no roles assigned**.\n\nThis could mean:\n1. The user 'diana' exists but has no roles granted to them\n2. The user 'diana' doesn't exist in the system\n3. The user has direct database permissions instead of role-based permissions\n\nWould you like me to:\n- Check what **direct database permissions** user diana has (using `sec_userDbPermissions`)?\n- Verify if the user exists by checking session information?\n- Look for a different user name (in case of a typo)?", - "metric_reasons": [], - "recommendation": null, - "turn_details": [ - { - "turn": 1, - "input": "Show me the user roles", - "mode": "clarification", - "passed": true, - "actual_tools": [] - }, - { - "turn": 2, - "input": "Which roles are assigned to user diana?", - "mode": "tool", - "passed": true, - "actual_tools": [ - { - "name": "sec_userRoles", - "params": { - "user_name": "diana" - } - } - ] - } - ] - }, - { - "case_id": "sec_rolePermissions_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which role, then calls sec_rolePermissions after user clarifies", - "input": "Turn 1: What access rights does the role have? | Turn 2: What database access has the analyst role been granted?", - "expected_tools": [ - { - "name": "sec_rolePermissions", - "params": { - "role_name": "analyst" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_rolePermissions", - "params": { - "role_name": "analyst" - } - } - ], - "actual_output": "The query completed successfully, but **no database access permissions were found for the \"analyst\" role**. This could mean:\n\n1. **The role doesn't exist** - There may not be a role named \"analyst\" in the system\n2. **The role has no permissions** - The role exists but hasn't been granted any database access rights yet\n3. **Case sensitivity** - The role name might have different casing (e.g., \"Analyst\", \"ANALYST\")\n\nWould you like me to:\n- Check if the role exists with a different case variation?\n- List all available roles in the system?\n- Check permissions for a different role name?\n\nPlease let me know how you'd like to proceed!", - "metric_reasons": [], - "recommendation": null, - "turn_details": [ - { - "turn": 1, - "input": "What access rights does the role have?", - "mode": "clarification", - "passed": true, - "actual_tools": [] - }, - { - "turn": 2, - "input": "What database access has the analyst role been granted?", - "mode": "tool", - "passed": true, - "actual_tools": [ - { - "name": "sec_rolePermissions", - "params": { - "role_name": "analyst" - } - } - ] - } - ] - }, - { - "case_id": "sec_userDbPermissions_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which user, then calls sec_userDbPermissions after user clarifies", - "input": "Turn 1: Show me the database permissions | Turn 2: What databases can user bob access and what can they do in each?", - "expected_tools": [ - { - "name": "sec_userDbPermissions", - "params": { - "user_name": "bob" - } - } - ], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [ - { - "name": "sec_userDbPermissions", - "params": { - "user_name": "bob" - } - } - ], - "actual_output": "The query results show that user **'bob'** has **no direct database permissions** granted in the system. This means:\n\n- No direct SELECT, INSERT, UPDATE, DELETE, or other access rights are assigned to 'bob' at the database or table level\n- Bob may still have access through **role-based permissions** (roles assigned to the user that have permissions)\n\n**Would you like me to check:**\n1. **What roles are assigned to bob?** (using `sec_userRoles`) - This would show if bob has any roles that might grant permissions\n2. **The permissions of specific roles** that bob might belong to?\n\nThis would give us a complete picture of bob's effective access rights in the system.", - "metric_reasons": [], - "recommendation": null, - "turn_details": [ - { - "turn": 1, - "input": "Show me the database permissions", - "mode": "clarification", - "passed": true, - "actual_tools": [] - }, - { - "turn": 2, - "input": "What databases can user bob access and what can they do in each?", - "mode": "tool", - "passed": true, - "actual_tools": [ - { - "name": "sec_userDbPermissions", - "params": { - "user_name": "bob" - } - } - ] - } - ] - }, - { - "case_id": "sec_unknown_tool_missing", - "case_type": "missing_parameter", - "description": "Vague access question with no user or role named \u2014 agent should ask for clarification", - "input": "What access does this user have in the system?", - "expected_tools": [], - "passed": true, - "failure_stage": null, - "failure_detail": null, - "actual_tools": [], - "actual_output": "I'd be happy to help you check what access a user has in the system. However, I need to know which user you'd like me to look up.\n\nCould you please provide the username you want to check?\n\nOnce you give me the username, I can show you:\n- **Database permissions** - What access rights (SELECT, INSERT, UPDATE, DELETE, etc.) the user has across databases\n- **Role assignments** - Which roles are assigned to the user\n- **Role permissions** - What access those roles provide", - "metric_reasons": [], - "recommendation": null, - "turn_details": null - } - ] -} diff --git a/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.md b/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.md deleted file mode 100644 index d0fc217..0000000 --- a/tests/eval_test/results/runs/2026-06-23T21-14-19.042368Z__all__baseline/summary.md +++ /dev/null @@ -1,268 +0,0 @@ -# Teradata MCP Eval Run Summary - -**Started (UTC):** 2026-06-23T21:14:19.042368+00:00 -**Module filter:** all -**Case type filter:** all -**Agent model:** us.anthropic.claude-sonnet-4-5-20250929-v1:0 -**Judge model:** us.anthropic.claude-sonnet-4-5-20250929-v1:0 -**Eval database:** demo_user -**Tool descriptions:** live MCP server (baseline) - -## Overview - -| Metric | Count | -| --- | ---: | -| Total cases | 106 | -| Passed | 103 | -| Failed | 3 | - -## Failed cases - -### base_tableList_missing_database (missing_parameter) - -**Description:** User asks for tables without specifying a database - -**Eval prompt:** - -> Show me what tables are available - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -- `base_tableList` with params: -```json -{ - "database_name": "" -} -``` - -**Agent response (excerpt):** - -> I can see the system has many tables available. The results show tables across multiple databases. Here's a summary of what I found: - -**Main Databases with Tables:** -- **DBC** - System database with metadata tables (InDoubtLog, StatsTbl, QryLog, DBCInfo, etc.) -- **TD_SYSFNLIB** - System function library -- **SYSLIB** - System library -- **TD_SERVER_DB** - Server database -- **SQLJ** - SQL/Java support -- **TD_SYSGPL** - System GPL (General Purpose Library) -- **SystemFe** - System functions -- **SYSSP - -**Failure (deterministic):** expected no tool calls for missing_parameter case, got ['base_tableList'] - -**Recommendation:** - -The prompt deliberately omits required information: "Show me what tables are available". The agent called `base_tableList` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response. - -### base_tableList_clarify_then_call (missing_parameter) - -**Description:** Agent asks which database, then calls base_tableList after user clarifies - -**Eval prompt:** - -> Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -- `base_tableList` with params: -```json -{ - "database_name": "" -} -``` - -**Agent response (excerpt):** - -> I can see there are tables available in the system. The results show many tables from the **DBC** database (which is the Teradata Data Dictionary/system catalog). - -Would you like me to: -1. **Filter by a specific database** - If you're looking for user/application tables rather than system tables, please tell me which database to show -2. **Show only user databases** - I can list the available databases first so you can pick one -3. **Continue showing all tables** - though the full list is quite l - -**Failure (deterministic):** turn 1: expected no tool calls for missing_parameter case, got ['base_tableList'] - -**Turn details:** - -```json -[ - { - "turn": 1, - "input": "Show me what tables are available", - "mode": "clarification", - "passed": false, - "failure_stage": "deterministic", - "failure_detail": "expected no tool calls for missing_parameter case, got ['base_tableList']", - "actual_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "" - } - } - ] - } -] -``` - -**Recommendation:** - -The prompt deliberately omits required information: "(multi-turn — see turn details)". The agent called `base_tableList` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response. - -### qlty_distinctCategories_missing_column (missing_parameter) - -**Description:** Agent must ask which column when none is specified - -**Eval prompt:** - -> What distinct values exist in a column of demo_user.evals_orders? - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -- `base_columnDescription` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Agent response (excerpt):** - -> The `demo_user.evals_orders` table has the following columns: - -1. **product_category** (VARCHAR) -2. **ship_date** (DATE) -3. **customer_name** (VARCHAR) -4. **order_id** (INTEGER) -5. **quantity** (INTEGER) -6. **amount** (DECIMAL) -7. **order_date** (DATE) - -Which column would you like to see the distinct values for? - -**Failure (deterministic):** expected no tool calls for missing_parameter case, got ['base_columnDescription'] - -**Recommendation:** - -The prompt deliberately omits required information: "What distinct values exist in a column of demo_user.evals_orders?". The agent called `base_columnDescription` instead of asking for clarification. Recommendation: review whether those tool descriptions invite calls without full context, and keep the eval prompt vague enough that asking is the only reasonable response. - -## Passed cases - -- `base_readQuery_happy` (happy_path) -- `base_readQuery_ambiguous_vs_tablePreview` (ambiguous_selection) -- `base_readQuery_missing_sql` (missing_parameter) -- `base_tableList_happy` (happy_path) -- `base_tableDDL_happy` (happy_path) -- `base_tableDDL_ambiguous_vs_columnMetadata` (ambiguous_selection) -- `base_columnMetadata_happy` (happy_path) -- `base_columnMetadata_ambiguous_vs_tableDDL` (ambiguous_selection) -- `base_tablePreview_happy` (happy_path) -- `base_tablePreview_ambiguous_vs_readQuery` (ambiguous_selection) -- `base_databaseList_happy` (happy_path) -- `base_tableList_ambiguous_vs_databaseList` (ambiguous_selection) -- `base_databaseList_ambiguous_vs_tableList` (ambiguous_selection) -- `base_saveDDL_ambiguous_vs_tableDDL` (ambiguous_selection) -- `base_tableAffinity_ambiguous_vs_tableUsage` (ambiguous_selection) -- `base_tableUsage_ambiguous_vs_tableAffinity` (ambiguous_selection) -- `base_saveDDL_happy` (happy_path) -- `base_tableAffinity_happy` (happy_path) -- `base_tableUsage_happy` (happy_path) -- `base_multi_tool_list_then_preview` (multi_tool) -- `base_multi_tool_ddl_then_columns` (multi_tool) -- `base_tableDDL_missing_table` (missing_parameter) -- `base_tablePreview_missing_table` (missing_parameter) -- `base_saveDDL_missing_table` (missing_parameter) -- `base_tablePreview_clarify_then_call` (missing_parameter) -- `base_readQuery_clarify_then_call` (missing_parameter) -- `base_columnDescription_happy` (happy_path) -- `base_columnDescription_ambiguous_vs_columnMetadata` (ambiguous_selection) -- `base_columnMetadata_ambiguous_vs_columnDescription` (ambiguous_selection) -- `base_unknown_tool_missing` (missing_parameter) -- `chat_completeChat_missing_message` (missing_parameter) -- `chat_unknown_tool_missing` (missing_parameter) -- `dba_tableSpace_happy` (happy_path) -- `dba_tableSqlList_happy` (happy_path) -- `dba_userSqlList_happy` (happy_path) -- `dba_databaseSpace_happy` (happy_path) -- `dba_tableUsageImpact_happy` (happy_path) -- `dba_resusageSummary_happy` (happy_path) -- `dba_databaseVersion_happy` (happy_path) -- `dba_flowControl_happy` (happy_path) -- `dba_featureUsage_happy` (happy_path) -- `dba_userDelay_happy` (happy_path) -- `dba_sessionInfo_happy` (happy_path) -- `dba_systemSpace_happy` (happy_path) -- `dba_tableSpace_ambiguous_vs_databaseSpace` (ambiguous_selection) -- `dba_databaseSpace_ambiguous_vs_systemSpace` (ambiguous_selection) -- `dba_systemSpace_ambiguous_vs_databaseSpace` (ambiguous_selection) -- `dba_tableSqlList_ambiguous_vs_userSqlList` (ambiguous_selection) -- `dba_userSqlList_ambiguous_vs_tableSqlList` (ambiguous_selection) -- `dba_tableUsageImpact_ambiguous_vs_resusageSummary` (ambiguous_selection) -- `dba_resusageSummary_ambiguous_vs_tableUsageImpact` (ambiguous_selection) -- `dba_tableUsageImpact_ambiguous_vs_sessionInfo` (ambiguous_selection) -- `dba_sessionInfo_ambiguous_vs_tableUsageImpact` (ambiguous_selection) -- `dba_userDelay_ambiguous_vs_flowControl` (ambiguous_selection) -- `dba_flowControl_ambiguous_vs_userDelay` (ambiguous_selection) -- `dba_tableSpace_missing_db` (missing_parameter) -- `dba_tableSqlList_missing_table` (missing_parameter) -- `dba_userSqlList_missing_user` (missing_parameter) -- `dba_databaseSpace_missing_db` (missing_parameter) -- `dba_tableUsageImpact_missing_db` (missing_parameter) -- `dba_databaseSpace_clarify_then_call` (missing_parameter) -- `dba_tableSqlList_clarify_then_call` (missing_parameter) -- `dba_userSqlList_clarify_then_call` (missing_parameter) -- `dba_unknown_tool_missing` (missing_parameter) -- `plot_line_chart_happy` (happy_path) -- `plot_pie_chart_happy` (happy_path) -- `plot_polar_chart_happy` (happy_path) -- `plot_radar_chart_happy` (happy_path) -- `plot_line_vs_radar_ambiguous` (ambiguous_selection) -- `plot_pie_vs_polar_ambiguous` (ambiguous_selection) -- `plot_missing_data` (missing_parameter) -- `plot_pie_chart_direct` (happy_path) -- `plot_missing_table` (missing_parameter) -- `plot_unknown_tool_missing` (missing_parameter) -- `qlty_missingValues_happy` (happy_path) -- `qlty_missingValues_ambiguous_vs_rowsWithMissingValues` (ambiguous_selection) -- `qlty_negativeValues_happy` (happy_path) -- `qlty_distinctCategories_happy` (happy_path) -- `qlty_standardDeviation_happy` (happy_path) -- `qlty_standardDeviation_ambiguous_vs_univariateStatistics` (ambiguous_selection) -- `qlty_columnSummary_happy` (happy_path) -- `qlty_univariateStatistics_happy` (happy_path) -- `qlty_univariateStatistics_ambiguous_vs_columnSummary` (ambiguous_selection) -- `qlty_rowsWithMissingValues_happy` (happy_path) -- `qlty_rowsWithMissingValues_ambiguous_vs_missingValues` (ambiguous_selection) -- `qlty_multi_tool_summary_then_nulls` (multi_tool) -- `qlty_unknown_tool_missing` (missing_parameter) -- `sec_userDbPermissions_ambiguous_vs_userRoles` (ambiguous_selection) -- `sec_userDbPermissions_missing_user` (missing_parameter) -- `sec_rolePermissions_ambiguous_vs_userDbPermissions` (ambiguous_selection) -- `sec_userRoles_ambiguous_vs_rolePermissions` (ambiguous_selection) -- `sec_rolePermissions_ambiguous_vs_userRoles` (ambiguous_selection) -- `sec_userRoles_ambiguous_vs_userDbPermissions` (ambiguous_selection) -- `sec_multi_tool_user_roles_then_role_permissions` (multi_tool) -- `sec_userDbPermissions_happy` (happy_path) -- `sec_rolePermissions_happy` (happy_path) -- `sec_userRoles_happy` (happy_path) -- `sec_rolePermissions_missing_role` (missing_parameter) -- `sec_userRoles_missing_user` (missing_parameter) -- `sec_userRoles_clarify_then_call` (missing_parameter) -- `sec_rolePermissions_clarify_then_call` (missing_parameter) -- `sec_userDbPermissions_clarify_then_call` (missing_parameter) -- `sec_unknown_tool_missing` (missing_parameter) - diff --git a/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/manifest.json b/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/manifest.json deleted file mode 100644 index 7f7bf6a..0000000 --- a/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/manifest.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "run_id": "2026-06-24T00-05-13.720459Z__base__baseline", - "started_at": "2026-06-24T00:05:13.720459+00:00", - "module_filter": "base", - "case_type_filter": "all", - "run_label": null, - "description_mode": "mcp_server", - "description_overrides_file": null, - "description_override_count": 0, - "agent_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "judge_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "evals_database": "demo_user", - "total": 32, - "passed": 0, - "failed": 32, - "artifacts": { - "summary_md": "summary.md", - "summary_json": "summary.json", - "manifest": "manifest.json", - "suggested_overrides": "suggested_overrides.json" - } -} diff --git a/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.json b/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.json deleted file mode 100644 index f537d98..0000000 --- a/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.json +++ /dev/null @@ -1,683 +0,0 @@ -{ - "run_id": "2026-06-24T00-05-13.720459Z__base__baseline", - "started_at": "2026-06-24T00:05:13.720459+00:00", - "module_filter": "base", - "case_type_filter": "all", - "run_label": null, - "agent_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "judge_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "evals_database": "demo_user", - "description_mode": "mcp_server", - "description_overrides_file": null, - "description_override_count": 0, - "total": 32, - "passed": 0, - "failed": 32, - "cases": [ - { - "case_id": "base_readQuery_happy", - "case_type": "happy_path", - "description": "Execute a SQL query against the evals employees table", - "input": "Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'", - "expected_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_readQuery_ambiguous_vs_tablePreview", - "case_type": "ambiguous_selection", - "description": "Prompt with a WHERE clause should use base_readQuery, not base_tablePreview", - "input": "Get me all orders from demo_user.evals_orders where the amount is greater than 500", - "expected_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_readQuery_missing_sql", - "case_type": "missing_parameter", - "description": "Agent must ask for clarification when no SQL or table is given", - "input": "Query the database for me", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_happy", - "case_type": "happy_path", - "description": "List tables in the evals database", - "input": "What tables are in the demo_user database?", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_missing_database", - "case_type": "missing_parameter", - "description": "User asks for tables without specifying a database", - "input": "Show me what tables are available", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableDDL_happy", - "case_type": "happy_path", - "description": "Retrieve DDL for the evals employees table", - "input": "Show me the CREATE statement for evals_employees in the demo_user database", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableDDL_ambiguous_vs_columnMetadata", - "case_type": "ambiguous_selection", - "description": "Asking for full structure including constraints should prefer base_tableDDL", - "input": "Describe the full structure of demo_user.evals_employees including any constraints and indexes", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnMetadata_happy", - "case_type": "happy_path", - "description": "Get precise Teradata type codes, character sets, and precision for columns in evals_orders", - "input": "Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnMetadata_ambiguous_vs_tableDDL", - "case_type": "ambiguous_selection", - "description": "Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL", - "input": "I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_happy", - "case_type": "happy_path", - "description": "Preview a sample of rows from the evals orders table", - "input": "Give me a quick preview of what's in demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_ambiguous_vs_readQuery", - "case_type": "ambiguous_selection", - "description": "A simple 'show me a few rows' without a WHERE clause should use base_tablePreview", - "input": "Show me a few rows from demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_databaseList_happy", - "case_type": "happy_path", - "description": "List all accessible databases", - "input": "What databases do I have access to?", - "expected_tools": [ - { - "name": "base_databaseList", - "params": {} - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_ambiguous_vs_databaseList", - "case_type": "ambiguous_selection", - "description": "Listing tables inside one database should prefer base_tableList, not base_databaseList", - "input": "What tables exist inside the demo_user database?", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_databaseList_ambiguous_vs_tableList", - "case_type": "ambiguous_selection", - "description": "Listing accessible databases should prefer base_databaseList, not base_tableList", - "input": "Which databases am I allowed to connect to on this system?", - "expected_tools": [ - { - "name": "base_databaseList", - "params": {} - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_saveDDL_ambiguous_vs_tableDDL", - "case_type": "ambiguous_selection", - "description": "Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL", - "input": "Export the DDL for demo_user.evals_orders to a file on disk", - "expected_tools": [ - { - "name": "base_saveDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableAffinity_ambiguous_vs_tableUsage", - "case_type": "ambiguous_selection", - "description": "Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage", - "input": "Which tables in demo_user tend to appear in the same queries as evals_employees?", - "expected_tools": [ - { - "name": "base_tableAffinity", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableUsage_ambiguous_vs_tableAffinity", - "case_type": "ambiguous_selection", - "description": "Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity", - "input": "Who has been reading tables in demo_user lately and how often?", - "expected_tools": [ - { - "name": "base_tableUsage", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_saveDDL_happy", - "case_type": "happy_path", - "description": "Save DDL for the evals employees table to a file", - "input": "Save the DDL for evals_employees in demo_user to a file", - "expected_tools": [ - { - "name": "base_saveDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableAffinity_happy", - "case_type": "happy_path", - "description": "Find tables commonly queried together with evals_orders", - "input": "Which tables in demo_user are most often queried together with evals_orders?", - "expected_tools": [ - { - "name": "base_tableAffinity", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableUsage_happy", - "case_type": "happy_path", - "description": "Check usage metrics for the evals database", - "input": "How frequently are the tables in demo_user being accessed and by whom?", - "expected_tools": [ - { - "name": "base_tableUsage", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_multi_tool_list_then_preview", - "case_type": "multi_tool", - "description": "List tables in evals database then preview one", - "input": "List all tables in demo_user, then show me a preview of evals_employees", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - }, - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_multi_tool_ddl_then_columns", - "case_type": "multi_tool", - "description": "Get DDL then drill into column detail for the same table", - "input": "First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - }, - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableDDL_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask for database name when only table is provided", - "input": "Show me the CREATE statement for evals_employees", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask which table to preview", - "input": "Preview some rows", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_saveDDL_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask which table to save DDL for", - "input": "Save the DDL to a file", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which table to preview, then calls base_tablePreview after user clarifies", - "input": "Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which database, then calls base_tableList after user clarifies", - "input": "Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_readQuery_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks for SQL, then calls base_readQuery after user provides the query", - "input": "Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnDescription_happy", - "case_type": "happy_path", - "description": "Show basic column info for the evals employees table", - "input": "Show me the columns in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnDescription_ambiguous_vs_columnMetadata", - "case_type": "ambiguous_selection", - "description": "A simple 'what columns does this table have' question should prefer base_columnDescription", - "input": "What are the column names and types in demo_user.evals_orders?", - "expected_tools": [ - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnMetadata_ambiguous_vs_columnDescription", - "case_type": "ambiguous_selection", - "description": "Asking for precise Teradata type codes and character sets should prefer base_columnMetadata", - "input": "Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_unknown_tool_missing", - "case_type": "missing_parameter", - "description": "User asks to do something no tool supports \u2014 agent should explain and ask for clarification", - "input": "Automatically tune my Teradata database indexes and rebalance the AMPs", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - } - ] -} diff --git a/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.md b/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.md deleted file mode 100644 index e153c05..0000000 --- a/tests/eval_test/results/runs/2026-06-24T00-05-13.720459Z__base__baseline/summary.md +++ /dev/null @@ -1,867 +0,0 @@ -# Teradata MCP Eval Run Summary - -**Started (UTC):** 2026-06-24T00:05:13.720459+00:00 -**Module filter:** base -**Case type filter:** all -**Agent model:** anthropic.claude-3-5-sonnet-20241022-v2:0 -**Judge model:** anthropic.claude-3-5-sonnet-20241022-v2:0 -**Eval database:** demo_user -**Tool descriptions:** live MCP server (baseline) - -## Overview - -| Metric | Count | -| --- | ---: | -| Total cases | 32 | -| Passed | 0 | -| Failed | 32 | - -## Failed cases - -### base_readQuery_happy (happy_path) - -**Description:** Execute a SQL query against the evals employees table - -**Eval prompt:** - -> Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales' - -**Expected tool(s):** - -- `base_readQuery` with params: -```json -{ - "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_readQuery_ambiguous_vs_tablePreview (ambiguous_selection) - -**Description:** Prompt with a WHERE clause should use base_readQuery, not base_tablePreview - -**Eval prompt:** - -> Get me all orders from demo_user.evals_orders where the amount is greater than 500 - -**Expected tool(s):** - -- `base_readQuery` with params: -```json -{ - "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_readQuery_missing_sql (missing_parameter) - -**Description:** Agent must ask for clarification when no SQL or table is given - -**Eval prompt:** - -> Query the database for me - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_happy (happy_path) - -**Description:** List tables in the evals database - -**Eval prompt:** - -> What tables are in the demo_user database? - -**Expected tool(s):** - -- `base_tableList` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_missing_database (missing_parameter) - -**Description:** User asks for tables without specifying a database - -**Eval prompt:** - -> Show me what tables are available - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableDDL_happy (happy_path) - -**Description:** Retrieve DDL for the evals employees table - -**Eval prompt:** - -> Show me the CREATE statement for evals_employees in the demo_user database - -**Expected tool(s):** - -- `base_tableDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableDDL_ambiguous_vs_columnMetadata (ambiguous_selection) - -**Description:** Asking for full structure including constraints should prefer base_tableDDL - -**Eval prompt:** - -> Describe the full structure of demo_user.evals_employees including any constraints and indexes - -**Expected tool(s):** - -- `base_tableDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnMetadata_happy (happy_path) - -**Description:** Get precise Teradata type codes, character sets, and precision for columns in evals_orders - -**Eval prompt:** - -> Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders - -**Expected tool(s):** - -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnMetadata_ambiguous_vs_tableDDL (ambiguous_selection) - -**Description:** Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL - -**Eval prompt:** - -> I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees - -**Expected tool(s):** - -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_happy (happy_path) - -**Description:** Preview a sample of rows from the evals orders table - -**Eval prompt:** - -> Give me a quick preview of what's in demo_user.evals_orders - -**Expected tool(s):** - -- `base_tablePreview` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_ambiguous_vs_readQuery (ambiguous_selection) - -**Description:** A simple 'show me a few rows' without a WHERE clause should use base_tablePreview - -**Eval prompt:** - -> Show me a few rows from demo_user.evals_orders - -**Expected tool(s):** - -- `base_tablePreview` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_databaseList_happy (happy_path) - -**Description:** List all accessible databases - -**Eval prompt:** - -> What databases do I have access to? - -**Expected tool(s):** - -- `base_databaseList` with params: -```json -{} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_ambiguous_vs_databaseList (ambiguous_selection) - -**Description:** Listing tables inside one database should prefer base_tableList, not base_databaseList - -**Eval prompt:** - -> What tables exist inside the demo_user database? - -**Expected tool(s):** - -- `base_tableList` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_databaseList_ambiguous_vs_tableList (ambiguous_selection) - -**Description:** Listing accessible databases should prefer base_databaseList, not base_tableList - -**Eval prompt:** - -> Which databases am I allowed to connect to on this system? - -**Expected tool(s):** - -- `base_databaseList` with params: -```json -{} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_saveDDL_ambiguous_vs_tableDDL (ambiguous_selection) - -**Description:** Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL - -**Eval prompt:** - -> Export the DDL for demo_user.evals_orders to a file on disk - -**Expected tool(s):** - -- `base_saveDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableAffinity_ambiguous_vs_tableUsage (ambiguous_selection) - -**Description:** Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage - -**Eval prompt:** - -> Which tables in demo_user tend to appear in the same queries as evals_employees? - -**Expected tool(s):** - -- `base_tableAffinity` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableUsage_ambiguous_vs_tableAffinity (ambiguous_selection) - -**Description:** Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity - -**Eval prompt:** - -> Who has been reading tables in demo_user lately and how often? - -**Expected tool(s):** - -- `base_tableUsage` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_saveDDL_happy (happy_path) - -**Description:** Save DDL for the evals employees table to a file - -**Eval prompt:** - -> Save the DDL for evals_employees in demo_user to a file - -**Expected tool(s):** - -- `base_saveDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableAffinity_happy (happy_path) - -**Description:** Find tables commonly queried together with evals_orders - -**Eval prompt:** - -> Which tables in demo_user are most often queried together with evals_orders? - -**Expected tool(s):** - -- `base_tableAffinity` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableUsage_happy (happy_path) - -**Description:** Check usage metrics for the evals database - -**Eval prompt:** - -> How frequently are the tables in demo_user being accessed and by whom? - -**Expected tool(s):** - -- `base_tableUsage` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_multi_tool_list_then_preview (multi_tool) - -**Description:** List tables in evals database then preview one - -**Eval prompt:** - -> List all tables in demo_user, then show me a preview of evals_employees - -**Expected tool(s):** - -- `base_tableList` with params: -```json -{ - "database_name": "demo_user" -} -``` -- `base_tablePreview` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_multi_tool_ddl_then_columns (multi_tool) - -**Description:** Get DDL then drill into column detail for the same table - -**Eval prompt:** - -> First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata - -**Expected tool(s):** - -- `base_tableDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableDDL_missing_table (missing_parameter) - -**Description:** Agent must ask for database name when only table is provided - -**Eval prompt:** - -> Show me the CREATE statement for evals_employees - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_missing_table (missing_parameter) - -**Description:** Agent must ask which table to preview - -**Eval prompt:** - -> Preview some rows - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_saveDDL_missing_table (missing_parameter) - -**Description:** Agent must ask which table to save DDL for - -**Eval prompt:** - -> Save the DDL to a file - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_clarify_then_call (missing_parameter) - -**Description:** Agent asks which table to preview, then calls base_tablePreview after user clarifies - -**Eval prompt:** - -> Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_clarify_then_call (missing_parameter) - -**Description:** Agent asks which database, then calls base_tableList after user clarifies - -**Eval prompt:** - -> Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_readQuery_clarify_then_call (missing_parameter) - -**Description:** Agent asks for SQL, then calls base_readQuery after user provides the query - -**Eval prompt:** - -> Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales' - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnDescription_happy (happy_path) - -**Description:** Show basic column info for the evals employees table - -**Eval prompt:** - -> Show me the columns in demo_user.evals_employees - -**Expected tool(s):** - -- `base_columnDescription` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnDescription_ambiguous_vs_columnMetadata (ambiguous_selection) - -**Description:** A simple 'what columns does this table have' question should prefer base_columnDescription - -**Eval prompt:** - -> What are the column names and types in demo_user.evals_orders? - -**Expected tool(s):** - -- `base_columnDescription` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnMetadata_ambiguous_vs_columnDescription (ambiguous_selection) - -**Description:** Asking for precise Teradata type codes and character sets should prefer base_columnMetadata - -**Eval prompt:** - -> Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees - -**Expected tool(s):** - -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_unknown_tool_missing (missing_parameter) - -**Description:** User asks to do something no tool supports — agent should explain and ask for clarification - -**Eval prompt:** - -> Automatically tune my Teradata database indexes and rebalance the AMPs - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -## Passed cases - -_None._ - diff --git a/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/manifest.json b/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/manifest.json deleted file mode 100644 index 6626fe8..0000000 --- a/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/manifest.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "run_id": "2026-06-24T00-06-59.222908Z__base__baseline", - "started_at": "2026-06-24T00:06:59.222908+00:00", - "module_filter": "base", - "case_type_filter": "all", - "run_label": null, - "description_mode": "mcp_server", - "description_overrides_file": null, - "description_override_count": 0, - "agent_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "judge_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "evals_database": "demo_user", - "total": 32, - "passed": 0, - "failed": 32, - "artifacts": { - "summary_md": "summary.md", - "summary_json": "summary.json", - "manifest": "manifest.json", - "suggested_overrides": "suggested_overrides.json" - } -} diff --git a/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json b/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json deleted file mode 100644 index 90bd6d6..0000000 --- a/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.json +++ /dev/null @@ -1,683 +0,0 @@ -{ - "run_id": "2026-06-24T00-06-59.222908Z__base__baseline", - "started_at": "2026-06-24T00:06:59.222908+00:00", - "module_filter": "base", - "case_type_filter": "all", - "run_label": null, - "agent_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "judge_model_id": "anthropic.claude-3-5-sonnet-20241022-v2:0", - "evals_database": "demo_user", - "description_mode": "mcp_server", - "description_overrides_file": null, - "description_override_count": 0, - "total": 32, - "passed": 0, - "failed": 32, - "cases": [ - { - "case_id": "base_readQuery_happy", - "case_type": "happy_path", - "description": "Execute a SQL query against the evals employees table", - "input": "Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'", - "expected_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_readQuery_ambiguous_vs_tablePreview", - "case_type": "ambiguous_selection", - "description": "Prompt with a WHERE clause should use base_readQuery, not base_tablePreview", - "input": "Get me all orders from demo_user.evals_orders where the amount is greater than 500", - "expected_tools": [ - { - "name": "base_readQuery", - "params": { - "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_readQuery_missing_sql", - "case_type": "missing_parameter", - "description": "Agent must ask for clarification when no SQL or table is given", - "input": "Query the database for me", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_happy", - "case_type": "happy_path", - "description": "List tables in the evals database", - "input": "What tables are in the demo_user database?", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_missing_database", - "case_type": "missing_parameter", - "description": "User asks for tables without specifying a database", - "input": "Show me what tables are available", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableDDL_happy", - "case_type": "happy_path", - "description": "Retrieve DDL for the evals employees table", - "input": "Show me the CREATE statement for evals_employees in the demo_user database", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableDDL_ambiguous_vs_columnMetadata", - "case_type": "ambiguous_selection", - "description": "Asking for full structure including constraints should prefer base_tableDDL", - "input": "Describe the full structure of demo_user.evals_employees including any constraints and indexes", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnMetadata_happy", - "case_type": "happy_path", - "description": "Get precise Teradata type codes, character sets, and precision for columns in evals_orders", - "input": "Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnMetadata_ambiguous_vs_tableDDL", - "case_type": "ambiguous_selection", - "description": "Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL", - "input": "I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_happy", - "case_type": "happy_path", - "description": "Preview a sample of rows from the evals orders table", - "input": "Give me a quick preview of what's in demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_ambiguous_vs_readQuery", - "case_type": "ambiguous_selection", - "description": "A simple 'show me a few rows' without a WHERE clause should use base_tablePreview", - "input": "Show me a few rows from demo_user.evals_orders", - "expected_tools": [ - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_databaseList_happy", - "case_type": "happy_path", - "description": "List all accessible databases", - "input": "What databases do I have access to?", - "expected_tools": [ - { - "name": "base_databaseList", - "params": {} - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_ambiguous_vs_databaseList", - "case_type": "ambiguous_selection", - "description": "Listing tables inside one database should prefer base_tableList, not base_databaseList", - "input": "What tables exist inside the demo_user database?", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_databaseList_ambiguous_vs_tableList", - "case_type": "ambiguous_selection", - "description": "Listing accessible databases should prefer base_databaseList, not base_tableList", - "input": "Which databases am I allowed to connect to on this system?", - "expected_tools": [ - { - "name": "base_databaseList", - "params": {} - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_saveDDL_ambiguous_vs_tableDDL", - "case_type": "ambiguous_selection", - "description": "Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL", - "input": "Export the DDL for demo_user.evals_orders to a file on disk", - "expected_tools": [ - { - "name": "base_saveDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableAffinity_ambiguous_vs_tableUsage", - "case_type": "ambiguous_selection", - "description": "Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage", - "input": "Which tables in demo_user tend to appear in the same queries as evals_employees?", - "expected_tools": [ - { - "name": "base_tableAffinity", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableUsage_ambiguous_vs_tableAffinity", - "case_type": "ambiguous_selection", - "description": "Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity", - "input": "Who has been reading tables in demo_user lately and how often?", - "expected_tools": [ - { - "name": "base_tableUsage", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_saveDDL_happy", - "case_type": "happy_path", - "description": "Save DDL for the evals employees table to a file", - "input": "Save the DDL for evals_employees in demo_user to a file", - "expected_tools": [ - { - "name": "base_saveDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableAffinity_happy", - "case_type": "happy_path", - "description": "Find tables commonly queried together with evals_orders", - "input": "Which tables in demo_user are most often queried together with evals_orders?", - "expected_tools": [ - { - "name": "base_tableAffinity", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableUsage_happy", - "case_type": "happy_path", - "description": "Check usage metrics for the evals database", - "input": "How frequently are the tables in demo_user being accessed and by whom?", - "expected_tools": [ - { - "name": "base_tableUsage", - "params": { - "database_name": "demo_user" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_multi_tool_list_then_preview", - "case_type": "multi_tool", - "description": "List tables in evals database then preview one", - "input": "List all tables in demo_user, then show me a preview of evals_employees", - "expected_tools": [ - { - "name": "base_tableList", - "params": { - "database_name": "demo_user" - } - }, - { - "name": "base_tablePreview", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_multi_tool_ddl_then_columns", - "case_type": "multi_tool", - "description": "Get DDL then drill into column detail for the same table", - "input": "First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata", - "expected_tools": [ - { - "name": "base_tableDDL", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - }, - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableDDL_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask for database name when only table is provided", - "input": "Show me the CREATE statement for evals_employees", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask which table to preview", - "input": "Preview some rows", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_saveDDL_missing_table", - "case_type": "missing_parameter", - "description": "Agent must ask which table to save DDL for", - "input": "Save the DDL to a file", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tablePreview_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which table to preview, then calls base_tablePreview after user clarifies", - "input": "Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_tableList_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks which database, then calls base_tableList after user clarifies", - "input": "Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_readQuery_clarify_then_call", - "case_type": "missing_parameter", - "description": "Agent asks for SQL, then calls base_readQuery after user provides the query", - "input": "Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales'", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnDescription_happy", - "case_type": "happy_path", - "description": "Show basic column info for the evals employees table", - "input": "Show me the columns in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnDescription_ambiguous_vs_columnMetadata", - "case_type": "ambiguous_selection", - "description": "A simple 'what columns does this table have' question should prefer base_columnDescription", - "input": "What are the column names and types in demo_user.evals_orders?", - "expected_tools": [ - { - "name": "base_columnDescription", - "params": { - "database_name": "demo_user", - "table_name": "evals_orders" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_columnMetadata_ambiguous_vs_columnDescription", - "case_type": "ambiguous_selection", - "description": "Asking for precise Teradata type codes and character sets should prefer base_columnMetadata", - "input": "Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees", - "expected_tools": [ - { - "name": "base_columnMetadata", - "params": { - "database_name": "demo_user", - "object_name": "evals_employees" - } - } - ], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - }, - { - "case_id": "base_unknown_tool_missing", - "case_type": "missing_parameter", - "description": "User asks to do something no tool supports \u2014 agent should explain and ask for clarification", - "input": "Automatically tune my Teradata database indexes and rebalance the AMPs", - "expected_tools": [], - "passed": false, - "failure_stage": "agent", - "failure_detail": "unhandled errors in a TaskGroup (1 sub-exception)", - "actual_tools": null, - "actual_output": null, - "metric_reasons": [], - "recommendation": "The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`).", - "turn_details": null - } - ] -} diff --git a/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md b/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md deleted file mode 100644 index 9973a9d..0000000 --- a/tests/eval_test/results/runs/2026-06-24T00-06-59.222908Z__base__baseline/summary.md +++ /dev/null @@ -1,867 +0,0 @@ -# Teradata MCP Eval Run Summary - -**Started (UTC):** 2026-06-24T00:06:59.222908+00:00 -**Module filter:** base -**Case type filter:** all -**Agent model:** anthropic.claude-3-5-sonnet-20241022-v2:0 -**Judge model:** anthropic.claude-3-5-sonnet-20241022-v2:0 -**Eval database:** demo_user -**Tool descriptions:** live MCP server (baseline) - -## Overview - -| Metric | Count | -| --- | ---: | -| Total cases | 32 | -| Passed | 0 | -| Failed | 32 | - -## Failed cases - -### base_readQuery_happy (happy_path) - -**Description:** Execute a SQL query against the evals employees table - -**Eval prompt:** - -> Run this query: SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales' - -**Expected tool(s):** - -- `base_readQuery` with params: -```json -{ - "sql": "SELECT name, department, salary FROM demo_user.evals_employees WHERE department = 'Sales'" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_readQuery_ambiguous_vs_tablePreview (ambiguous_selection) - -**Description:** Prompt with a WHERE clause should use base_readQuery, not base_tablePreview - -**Eval prompt:** - -> Get me all orders from demo_user.evals_orders where the amount is greater than 500 - -**Expected tool(s):** - -- `base_readQuery` with params: -```json -{ - "sql": "SELECT * FROM demo_user.evals_orders WHERE amount > 500" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_readQuery_missing_sql (missing_parameter) - -**Description:** Agent must ask for clarification when no SQL or table is given - -**Eval prompt:** - -> Query the database for me - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_happy (happy_path) - -**Description:** List tables in the evals database - -**Eval prompt:** - -> What tables are in the demo_user database? - -**Expected tool(s):** - -- `base_tableList` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_missing_database (missing_parameter) - -**Description:** User asks for tables without specifying a database - -**Eval prompt:** - -> Show me what tables are available - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableDDL_happy (happy_path) - -**Description:** Retrieve DDL for the evals employees table - -**Eval prompt:** - -> Show me the CREATE statement for evals_employees in the demo_user database - -**Expected tool(s):** - -- `base_tableDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableDDL_ambiguous_vs_columnMetadata (ambiguous_selection) - -**Description:** Asking for full structure including constraints should prefer base_tableDDL - -**Eval prompt:** - -> Describe the full structure of demo_user.evals_employees including any constraints and indexes - -**Expected tool(s):** - -- `base_tableDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnMetadata_happy (happy_path) - -**Description:** Get precise Teradata type codes, character sets, and precision for columns in evals_orders - -**Eval prompt:** - -> Give me the exact Teradata type codes, character set, and decimal precision for every column in demo_user.evals_orders - -**Expected tool(s):** - -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnMetadata_ambiguous_vs_tableDDL (ambiguous_selection) - -**Description:** Asking for technical Teradata column precision metadata should prefer base_columnMetadata over base_tableDDL - -**Eval prompt:** - -> I need the UNICODE vs LATIN character set and nullability for every column in demo_user.evals_employees - -**Expected tool(s):** - -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_happy (happy_path) - -**Description:** Preview a sample of rows from the evals orders table - -**Eval prompt:** - -> Give me a quick preview of what's in demo_user.evals_orders - -**Expected tool(s):** - -- `base_tablePreview` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_ambiguous_vs_readQuery (ambiguous_selection) - -**Description:** A simple 'show me a few rows' without a WHERE clause should use base_tablePreview - -**Eval prompt:** - -> Show me a few rows from demo_user.evals_orders - -**Expected tool(s):** - -- `base_tablePreview` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_databaseList_happy (happy_path) - -**Description:** List all accessible databases - -**Eval prompt:** - -> What databases do I have access to? - -**Expected tool(s):** - -- `base_databaseList` with params: -```json -{} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_ambiguous_vs_databaseList (ambiguous_selection) - -**Description:** Listing tables inside one database should prefer base_tableList, not base_databaseList - -**Eval prompt:** - -> What tables exist inside the demo_user database? - -**Expected tool(s):** - -- `base_tableList` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_databaseList_ambiguous_vs_tableList (ambiguous_selection) - -**Description:** Listing accessible databases should prefer base_databaseList, not base_tableList - -**Eval prompt:** - -> Which databases am I allowed to connect to on this system? - -**Expected tool(s):** - -- `base_databaseList` with params: -```json -{} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_saveDDL_ambiguous_vs_tableDDL (ambiguous_selection) - -**Description:** Persisting DDL to a file should prefer base_saveDDL, not base_tableDDL - -**Eval prompt:** - -> Export the DDL for demo_user.evals_orders to a file on disk - -**Expected tool(s):** - -- `base_saveDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableAffinity_ambiguous_vs_tableUsage (ambiguous_selection) - -**Description:** Finding tables co-queried with a specific table should prefer base_tableAffinity, not base_tableUsage - -**Eval prompt:** - -> Which tables in demo_user tend to appear in the same queries as evals_employees? - -**Expected tool(s):** - -- `base_tableAffinity` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableUsage_ambiguous_vs_tableAffinity (ambiguous_selection) - -**Description:** Access frequency for tables in a database should prefer base_tableUsage, not base_tableAffinity - -**Eval prompt:** - -> Who has been reading tables in demo_user lately and how often? - -**Expected tool(s):** - -- `base_tableUsage` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_saveDDL_happy (happy_path) - -**Description:** Save DDL for the evals employees table to a file - -**Eval prompt:** - -> Save the DDL for evals_employees in demo_user to a file - -**Expected tool(s):** - -- `base_saveDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableAffinity_happy (happy_path) - -**Description:** Find tables commonly queried together with evals_orders - -**Eval prompt:** - -> Which tables in demo_user are most often queried together with evals_orders? - -**Expected tool(s):** - -- `base_tableAffinity` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableUsage_happy (happy_path) - -**Description:** Check usage metrics for the evals database - -**Eval prompt:** - -> How frequently are the tables in demo_user being accessed and by whom? - -**Expected tool(s):** - -- `base_tableUsage` with params: -```json -{ - "database_name": "demo_user" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_multi_tool_list_then_preview (multi_tool) - -**Description:** List tables in evals database then preview one - -**Eval prompt:** - -> List all tables in demo_user, then show me a preview of evals_employees - -**Expected tool(s):** - -- `base_tableList` with params: -```json -{ - "database_name": "demo_user" -} -``` -- `base_tablePreview` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_multi_tool_ddl_then_columns (multi_tool) - -**Description:** Get DDL then drill into column detail for the same table - -**Eval prompt:** - -> First show me the DDL for demo_user.evals_orders, then give me the detailed column metadata - -**Expected tool(s):** - -- `base_tableDDL` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableDDL_missing_table (missing_parameter) - -**Description:** Agent must ask for database name when only table is provided - -**Eval prompt:** - -> Show me the CREATE statement for evals_employees - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_missing_table (missing_parameter) - -**Description:** Agent must ask which table to preview - -**Eval prompt:** - -> Preview some rows - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_saveDDL_missing_table (missing_parameter) - -**Description:** Agent must ask which table to save DDL for - -**Eval prompt:** - -> Save the DDL to a file - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tablePreview_clarify_then_call (missing_parameter) - -**Description:** Agent asks which table to preview, then calls base_tablePreview after user clarifies - -**Eval prompt:** - -> Turn 1: Preview some rows for me | Turn 2: Preview rows from demo_user.evals_employees - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_tableList_clarify_then_call (missing_parameter) - -**Description:** Agent asks which database, then calls base_tableList after user clarifies - -**Eval prompt:** - -> Turn 1: Show me what tables are available | Turn 2: List tables in the demo_user database - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_readQuery_clarify_then_call (missing_parameter) - -**Description:** Agent asks for SQL, then calls base_readQuery after user provides the query - -**Eval prompt:** - -> Turn 1: Run a query for me | Turn 2: Run: SELECT name, department FROM demo_user.evals_employees WHERE department = 'Sales' - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnDescription_happy (happy_path) - -**Description:** Show basic column info for the evals employees table - -**Eval prompt:** - -> Show me the columns in demo_user.evals_employees - -**Expected tool(s):** - -- `base_columnDescription` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnDescription_ambiguous_vs_columnMetadata (ambiguous_selection) - -**Description:** A simple 'what columns does this table have' question should prefer base_columnDescription - -**Eval prompt:** - -> What are the column names and types in demo_user.evals_orders? - -**Expected tool(s):** - -- `base_columnDescription` with params: -```json -{ - "database_name": "demo_user", - "table_name": "evals_orders" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_columnMetadata_ambiguous_vs_columnDescription (ambiguous_selection) - -**Description:** Asking for precise Teradata type codes and character sets should prefer base_columnMetadata - -**Eval prompt:** - -> Give me the exact Teradata type codes, character sets, and precision details for every column in demo_user.evals_employees - -**Expected tool(s):** - -- `base_columnMetadata` with params: -```json -{ - "database_name": "demo_user", - "object_name": "evals_employees" -} -``` - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -### base_unknown_tool_missing (missing_parameter) - -**Description:** User asks to do something no tool supports — agent should explain and ask for clarification - -**Eval prompt:** - -> Automatically tune my Teradata database indexes and rebalance the AMPs - -**Expected tool(s):** - -_none_ - -**Actual tool(s):** - -_none_ - -**Failure (agent):** unhandled errors in a TaskGroup (1 sub-exception) - -**Recommendation:** - -The agent loop failed before scoring: unhandled errors in a TaskGroup (1 sub-exception). Check MCP server connectivity, Bedrock credentials, and that the eval prompt is reachable with the current environment (`EVALS_DATABASE=demo_user`). - -## Passed cases - -_None._ -