Skip to content

Commit 795bbd1

Browse files
Dhanush Shettyclaude
Dhanush Shetty
and
claude
committed
Improve test-all-tools.ts: add TOOL_ARG_OVERRIDES + 30 field defaults
- Add 70+ tool-specific argument overrides for schema, insights, ML, catalog, quality, ITSM, BI, connector, and streaming tools - Update 30+ field-name defaults to match actual InMemory seed data IDs - Pass tool name to generateArgs() for per-tool override support Pass rate: 173/214 (80.8%) → 201/214 (93.9%). 9 agents at 100%. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 09b00b6 commit 795bbd1

2 files changed

Lines changed: 305 additions & 215 deletions

File tree

‎scripts/test-all-tools.ts‎

Lines changed: 106 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,67 @@ const AGENTS = [
7373
// dw-orchestration excluded: internal service, not an MCP agent
7474
];
7575

76+
// Per-tool argument overrides — when a tool needs specific arg shapes
77+
const TOOL_ARG_OVERRIDES: Record<string, Record<string, unknown>> = {
78+
// Schema tools
79+
assess_impact: { change: { id: 'chg-1', customerId: 'cust-1', source: 'snowflake', database: 'analytics', schema: 'public', table: 'orders', changeType: 'column_added', severity: 'non-breaking', details: { column: 'test_col', newType: 'TEXT' } } },
80+
apply_migration: { migration: { id: 'mig-1', changeId: 'chg-1', customerId: 'cust-1', status: 'pending', forwardSql: 'ALTER TABLE analytics.public.orders ADD COLUMN test_col TEXT', rollbackSql: 'ALTER TABLE analytics.public.orders DROP COLUMN test_col', affectedSystems: ['analytics'], backwardCompatible: true, estimatedEffortHours: 0.5 } },
81+
rollback_migration: { migration: { id: 'mig-1', changeId: 'chg-1', customerId: 'cust-1', status: 'deployed', forwardSql: 'ALTER TABLE analytics.public.orders ADD COLUMN test_col TEXT', rollbackSql: 'ALTER TABLE analytics.public.orders DROP COLUMN test_col', affectedSystems: ['analytics'], backwardCompatible: true, estimatedEffortHours: 0.5 } },
82+
get_schema_snapshot: { source: 'snowflake', database: 'ANALYTICS', schema: 'PUBLIC', table: 'ORDERS' },
83+
validate_schema_compatibility: { change: { id: 'chg-1', customerId: 'cust-1', source: 'snowflake', database: 'analytics', schema: 'public', table: 'orders', changeType: 'column_added', severity: 'non-breaking', details: { column: 'test_col', newType: 'TEXT' } } },
84+
// Insights tools
85+
query_data_nl: { question: 'What is total revenue by month?' },
86+
generate_insight: { sql: 'SELECT date, amount FROM revenue_daily', results: [{ date: '2025-01-01', amount: 1000 }, { date: '2025-02-01', amount: 1200 }, { date: '2025-03-01', amount: 1100 }] },
87+
export_insight: { results: [{ date: '2025-01-01', amount: 1000 }, { date: '2025-02-01', amount: 1200 }], format: 'csv' },
88+
// Context Catalog tools
89+
explain_table: { tableIdentifier: 'orders' },
90+
blast_radius_analysis: { assetId: 'tbl-1', changeType: 'column_drop', columnName: 'status' },
91+
correlate_metadata: { assetId: 'tbl-1' },
92+
import_tribal_knowledge: { entries: [{ content: 'The orders table is the source of truth for all order data', tags: ['orders', 'sot'] }] },
93+
update_business_rule: { ruleId: 'rule-1', description: 'Updated rule for testing' },
94+
// ML tools
95+
evaluate_model: { model_id: 'model-churn-xgb-001', dataset_id: 'ds-churn-001' },
96+
compare_experiments: { experiment_ids: ['exp-churn-001'] },
97+
log_metrics: { experiment_id: 'exp-churn-001', run_id: 'run-001', metrics: [{ name: 'accuracy', value: 0.95 }] },
98+
create_feature_pipeline: { name: 'test-pipeline', dataset_id: 'ds-churn-001', transforms: [{ column: 'age', type: 'normalize', outputColumn: 'age_normalized' }] },
99+
materialize_features: { pipeline_id: 'fp-churn-001' },
100+
detect_model_drift: { model_id: 'model-churn-xgb-001', baseline_dataset_id: 'ds-churn-001' },
101+
ab_test_models: { model_a_id: 'model-churn-xgb-001', model_b_id: 'model-revenue-lgbm-001', dataset_id: 'ds-churn-001' },
102+
model_serve: { model_id: 'model-churn-xgb-001', platform: 'seldon' },
103+
get_model_versions: { model_name: 'churn-predictor' },
104+
// Streaming
105+
get_stream_health: { topology: 'topo-orders-cdc' },
106+
// Connector-specific database overrides
107+
list_glue_tables: { database: 'analytics_db' },
108+
get_glue_table: { database: 'analytics_db', table: 'orders' },
109+
list_hive_tables: { database: 'default' },
110+
get_hive_table_schema: { database: 'default', table: 'customers' },
111+
list_om_tables: { database: 'warehouse_db' },
112+
get_om_table: { tableId: 'tbl-001' },
113+
// Nessie
114+
list_nessie_tables: { ref: 'main' },
115+
get_nessie_content: { ref: 'main', key: 'analytics.customers' },
116+
create_nessie_branch: { name: 'test-branch', from: 'main' },
117+
diff_nessie_refs: { from: 'main', to: 'develop' },
118+
// Dataplex
119+
list_dataplex_entities: { zone: 'projects/my-project/locations/us-central1/lakes/analytics-lake/zones/curated-zone' },
120+
get_dataplex_entity: { name: 'projects/my-project/locations/us-central1/lakes/analytics-lake/zones/curated-zone/entities/orders' },
121+
// DataHub
122+
get_datahub_dataset: { urn: 'urn:li:dataset:(urn:li:dataPlatform:snowflake,analytics.public.orders,PROD)' },
123+
get_datahub_lineage: { urn: 'urn:li:dataset:(urn:li:dataPlatform:snowflake,analytics.public.orders,PROD)', direction: 'downstream' },
124+
// Purview
125+
get_purview_entity: { guid: 'pv-001' },
126+
// OpenLineage
127+
get_lineage_graph: { namespace: 'default', jobName: 'etl_orders' },
128+
list_lineage_datasets: { namespace: 'default' },
129+
// Catalog registry
130+
get_table_from_any_catalog: { provider: 'snowflake', database: 'ANALYTICS', schema: 'PUBLIC', table: 'ORDERS' },
131+
};
132+
76133
// Default args for common required fields
77-
function generateArgs(inputSchema: Record<string, unknown>): Record<string, unknown> {
78-
const args: Record<string, unknown> = { customerId: 'cust-1' };
134+
function generateArgs(toolName: string, inputSchema: Record<string, unknown>): Record<string, unknown> {
135+
const overrides = TOOL_ARG_OVERRIDES[toolName] || {};
136+
const args: Record<string, unknown> = { customerId: 'cust-1', ...overrides };
79137
const properties = (inputSchema as any)?.properties || {};
80138
const required: string[] = (inputSchema as any)?.required || [];
81139

@@ -84,17 +142,17 @@ function generateArgs(inputSchema: Record<string, unknown>): Record<string, unkn
84142
const prop = properties[field] || {};
85143
switch (prop.type) {
86144
case 'string':
87-
// Smart defaults for known field names
145+
// Smart defaults for known field names — matched to actual seed data IDs
88146
if (field === 'datasetId') args[field] = 'orders';
89147
else if (field === 'pipelineId') args[field] = 'test-pipe';
90148
else if (field === 'incidentId') args[field] = 'inc-test';
91149
else if (field === 'migrationId') args[field] = 'mig-test';
92-
else if (field === 'streamId') args[field] = 'stream-test';
150+
else if (field === 'streamId') args[field] = 'topo-orders-cdc';
93151
else if (field === 'metricName') args[field] = 'revenue';
94152
else if (field === 'resource') args[field] = 'orders';
95153
else if (field === 'action') args[field] = 'read';
96154
else if (field === 'agentId') args[field] = 'dw-pipelines';
97-
else if (field === 'userId') args[field] = 'user-1';
155+
else if (field === 'userId') args[field] = 'user-001';
98156
else if (field === 'accessLevel') args[field] = 'read';
99157
else if (field === 'justification') args[field] = 'testing';
100158
else if (field === 'role') args[field] = 'viewer';
@@ -103,27 +161,53 @@ function generateArgs(inputSchema: Record<string, unknown>): Record<string, unkn
103161
else if (field === 'sourceDialect' || field === 'sourceSystem') args[field] = 'postgresql';
104162
else if (field === 'targetDialect' || field === 'targetSystem') args[field] = 'snowflake';
105163
else if (field === 'incidentType') args[field] = 'schema_change';
106-
else if (field === 'database') args[field] = 'test_db';
107-
else if (field === 'schema') args[field] = 'public';
164+
else if (field === 'database') args[field] = 'ANALYTICS';
165+
else if (field === 'schema') args[field] = 'PUBLIC';
108166
else if (field === 'table') args[field] = 'orders';
109167
else if (field === 'connectorId') args[field] = 'snowflake';
110168
else if (field === 'tableIdentifier') args[field] = 'orders';
111-
else if (field === 'assetId') args[field] = 'orders';
169+
else if (field === 'assetId') args[field] = 'tbl-1';
112170
else if (field === 'sourceDatasetId') args[field] = 'orders';
113171
else if (field === 'targetDatasetId') args[field] = 'customers';
114172
else if (field === 'changeType') args[field] = 'schema_change';
115-
else if (field === 'entries') args[field] = [{ key: 'test', value: 'test' }];
116-
else if (field === 'ruleId') args[field] = 'test-rule';
173+
else if (field === 'ruleId') args[field] = 'rule-1';
117174
else if (field === 'model_id' || field === 'modelId') args[field] = 'model-churn-xgb-001';
118175
else if (field === 'dataset_id') args[field] = 'ds-churn-001';
119-
else if (field === 'datasetId') args[field] = 'orders';
120176
else if (field === 'query') args[field] = 'SELECT * FROM orders LIMIT 10';
121-
else if (field === 'results') args[field] = [{ column: 'id', value: 1 }];
122-
else if (field === 'migrationSql') args[field] = 'ALTER TABLE orders ADD COLUMN test TEXT';
123-
else if (field === 'change') args[field] = { type: 'column_added', column: 'test', dataType: 'TEXT' };
124-
else if (field === 'experimentId') args[field] = 'exp-churn-001';
177+
else if (field === 'migrationSql') args[field] = 'ALTER TABLE orders ADD COLUMN test_col TEXT';
178+
else if (field === 'experimentId' || field === 'experiment_id') args[field] = 'exp-churn-001';
125179
else if (field === 'featurePipelineId') args[field] = 'fp-churn-001';
126180
else if (field === 'deploymentId') args[field] = 'deploy-churn-001';
181+
else if (field === 'dagId') args[field] = 'daily_pipeline';
182+
else if (field === 'runId') args[field] = 'run-001';
183+
else if (field === 'alertId') args[field] = 'alert-001';
184+
else if (field === 'subject') args[field] = 'orders-value';
185+
else if (field === 'traceId') args[field] = 'trace-001';
186+
else if (field === 'groupId') args[field] = 'group-001';
187+
else if (field === 'guid') args[field] = 'pv-001';
188+
else if (field === 'urn') args[field] = 'urn:li:dataset:(urn:li:dataPlatform:snowflake,analytics.public.orders,PROD)';
189+
else if (field === 'direction') args[field] = 'downstream';
190+
else if (field === 'namespace') args[field] = 'default';
191+
else if (field === 'jobName') args[field] = 'etl_orders';
192+
else if (field === 'ref') args[field] = 'main';
193+
else if (field === 'from') args[field] = 'main';
194+
else if (field === 'to') args[field] = 'develop';
195+
else if (field === 'key') args[field] = 'analytics.customers';
196+
else if (field === 'catalog') args[field] = 'main';
197+
else if (field === 'provider') args[field] = 'snowflake';
198+
else if (field === 'tableId') args[field] = 'tbl-001';
199+
else if (field === 'topology') args[field] = 'topo-orders-cdc';
200+
else if (field === 'pipeline_id') args[field] = 'fp-churn-001';
201+
else if (field === 'baseline_dataset_id') args[field] = 'ds-churn-001';
202+
else if (field === 'model_a_id') args[field] = 'model-churn-xgb-001';
203+
else if (field === 'model_b_id') args[field] = 'model-revenue-lgbm-001';
204+
else if (field === 'model_name') args[field] = 'churn-predictor';
205+
else if (field === 'suiteId') args[field] = 'gx-suite-1';
206+
else if (field === 'dashboardId') args[field] = 'dash-001';
207+
else if (field === 'ticketId') args[field] = 'INC0001';
208+
else if (field === 'question') args[field] = 'What is total revenue by month?';
209+
else if (field === 'format') args[field] = 'csv';
210+
else if (field === 'platform') args[field] = 'seldon';
127211
else args[field] = 'test';
128212
break;
129213
case 'number':
@@ -137,11 +221,17 @@ function generateArgs(inputSchema: Record<string, unknown>): Record<string, unkn
137221
if (field === 'affectedResources') args[field] = ['orders'];
138222
else if (field === 'rules') args[field] = [{ metric: 'freshness', operator: 'lte', threshold: 24, severity: 'warning', description: 'test' }];
139223
else if (field === 'metrics') args[field] = [{ name: 'accuracy', value: 0.95 }];
224+
else if (field === 'results') args[field] = [{ date: '2025-01-01', amount: 1000 }, { date: '2025-02-01', amount: 1200 }];
225+
else if (field === 'entries') args[field] = [{ content: 'The orders table is the source of truth', tags: ['orders'] }];
226+
else if (field === 'transforms') args[field] = [{ column: 'age', type: 'normalize', outputColumn: 'age_normalized' }];
227+
else if (field === 'experiment_ids') args[field] = ['exp-churn-001'];
140228
else args[field] = [];
141229
break;
142230
case 'object':
143231
if (field === 'pipelineSpec') args[field] = { name: 'test', tasks: [], dag: { nodes: [], edges: [] } };
144232
else if (field === 'config') args[field] = {};
233+
else if (field === 'change') args[field] = { id: 'chg-1', customerId: 'cust-1', source: 'snowflake', database: 'analytics', schema: 'public', table: 'orders', changeType: 'column_added', severity: 'non-breaking', details: { column: 'test_col', newType: 'TEXT' } };
234+
else if (field === 'migration') args[field] = { id: 'mig-1', changeId: 'chg-1', customerId: 'cust-1', status: 'pending', forwardSql: 'ALTER TABLE orders ADD COLUMN test_col TEXT', rollbackSql: 'ALTER TABLE orders DROP COLUMN test_col', affectedSystems: ['analytics'], backwardCompatible: true, estimatedEffortHours: 0.5 };
145235
else args[field] = {};
146236
break;
147237
default:
@@ -200,7 +290,7 @@ async function testAgent(agentName: string): Promise<AgentResult> {
200290
durationMs: 0,
201291
};
202292

203-
const args = generateArgs(tool.inputSchema || {});
293+
const args = generateArgs(tool.name, tool.inputSchema || {});
204294
const start = performance.now();
205295

206296
try {

0 commit comments

Comments
 (0)