|
| 1 | +/** |
| 2 | + * Developer Experience (DX) scoring — evaluates MCP server quality |
| 3 | + * from a developer's perspective using data already collected during eval. |
| 4 | + * |
| 5 | + * Three sub-scores, no LLM calls needed: |
| 6 | + * 1. Schema Quality (40%) — are tool input schemas well-defined? |
| 7 | + * 2. Documentation (30%) — do tools have meaningful descriptions? |
| 8 | + * 3. Error Messages (30%) — are error messages helpful when tools fail? |
| 9 | + */ |
| 10 | + |
| 11 | +import type { ToolInfo } from "../protocols/base.js"; |
| 12 | +import type { TaskResult } from "./executor.js"; |
| 13 | + |
| 14 | +export interface DxSubScores { |
| 15 | + schemaQuality: number; // 0-100 |
| 16 | + documentation: number; // 0-100 |
| 17 | + errorMessages: number; // 0-100 |
| 18 | + overall: number; // 0-100 |
| 19 | +} |
| 20 | + |
| 21 | +/** |
| 22 | + * Compute the DX score from tools metadata and execution results. |
| 23 | + * Pure function — no LLM calls, no async, no side effects. |
| 24 | + */ |
| 25 | +export function computeDxScore( |
| 26 | + tools: ToolInfo[], |
| 27 | + taskResults: TaskResult[], |
| 28 | +): DxSubScores { |
| 29 | + const schemaQuality = scoreSchemaQuality(tools); |
| 30 | + const documentation = scoreDocumentation(tools); |
| 31 | + const errorMessages = scoreErrorMessages(taskResults); |
| 32 | + |
| 33 | + // Weighted combination: schema 40%, docs 30%, errors 30% |
| 34 | + const overall = Math.round( |
| 35 | + schemaQuality * 0.4 + documentation * 0.3 + errorMessages * 0.3, |
| 36 | + ); |
| 37 | + |
| 38 | + return { schemaQuality, documentation, errorMessages, overall }; |
| 39 | +} |
| 40 | + |
| 41 | +/** |
| 42 | + * Score how well tool input schemas are defined. |
| 43 | + * Checks: properties exist, types declared, descriptions present, required fields specified. |
| 44 | + */ |
| 45 | +export function scoreSchemaQuality(tools: ToolInfo[]): number { |
| 46 | + if (tools.length === 0) return 50; // Neutral for no tools |
| 47 | + |
| 48 | + let totalScore = 0; |
| 49 | + |
| 50 | + for (const tool of tools) { |
| 51 | + const schema = tool.inputSchema; |
| 52 | + let toolScore = 0; |
| 53 | + let checks = 0; |
| 54 | + |
| 55 | + // Check 1: Has properties defined (not empty schema) |
| 56 | + const properties = |
| 57 | + (schema.properties as Record<string, Record<string, unknown>>) || {}; |
| 58 | + const propCount = Object.keys(properties).length; |
| 59 | + checks++; |
| 60 | + if (propCount > 0) toolScore++; |
| 61 | + |
| 62 | + // Check 2: Properties have types declared |
| 63 | + if (propCount > 0) { |
| 64 | + const withType = Object.values(properties).filter( |
| 65 | + (p) => p.type !== undefined, |
| 66 | + ).length; |
| 67 | + checks++; |
| 68 | + toolScore += withType / propCount; |
| 69 | + } |
| 70 | + |
| 71 | + // Check 3: Properties have descriptions |
| 72 | + if (propCount > 0) { |
| 73 | + const withDesc = Object.values(properties).filter( |
| 74 | + (p) => typeof p.description === "string" && p.description.length > 0, |
| 75 | + ).length; |
| 76 | + checks++; |
| 77 | + toolScore += withDesc / propCount; |
| 78 | + } |
| 79 | + |
| 80 | + // Check 4: Required fields specified |
| 81 | + const required = schema.required as string[] | undefined; |
| 82 | + checks++; |
| 83 | + if (Array.isArray(required) && required.length > 0) toolScore++; |
| 84 | + |
| 85 | + totalScore += checks > 0 ? toolScore / checks : 0; |
| 86 | + } |
| 87 | + |
| 88 | + return Math.round((totalScore / tools.length) * 100); |
| 89 | +} |
| 90 | + |
| 91 | +/** |
| 92 | + * Score how well tools are documented. |
| 93 | + * Checks: description exists, is meaningful (>20 chars), explains behavior. |
| 94 | + */ |
| 95 | +export function scoreDocumentation(tools: ToolInfo[]): number { |
| 96 | + if (tools.length === 0) return 50; |
| 97 | + |
| 98 | + let totalScore = 0; |
| 99 | + |
| 100 | + for (const tool of tools) { |
| 101 | + let toolScore = 0; |
| 102 | + |
| 103 | + // Check 1: Has a non-empty description |
| 104 | + const desc = tool.description || ""; |
| 105 | + if (desc.length > 0) toolScore += 0.4; |
| 106 | + |
| 107 | + // Check 2: Description is meaningful (> 20 chars) |
| 108 | + if (desc.length > 20) toolScore += 0.3; |
| 109 | + |
| 110 | + // Check 3: Description explains behavior (contains action words) |
| 111 | + if (desc.length > 0) { |
| 112 | + const actionPatterns = |
| 113 | + /\b(returns?|creates?|gets?|sets?|lists?|reads?|writes?|deletes?|updates?|searches?|sends?|fetches?|generates?|computes?|validates?|converts?|handles?|manages?|provides?)\b/i; |
| 114 | + if (actionPatterns.test(desc)) toolScore += 0.3; |
| 115 | + } |
| 116 | + |
| 117 | + totalScore += toolScore; |
| 118 | + } |
| 119 | + |
| 120 | + return Math.round((totalScore / tools.length) * 100); |
| 121 | +} |
| 122 | + |
| 123 | +/** |
| 124 | + * Score error message quality from failed task runs. |
| 125 | + * Checks: errors are informative (not empty), structured (not raw stack traces). |
| 126 | + * If no failures occurred, return neutral score (70) — no error data to evaluate. |
| 127 | + */ |
| 128 | +export function scoreErrorMessages(taskResults: TaskResult[]): number { |
| 129 | + // Collect all error messages from failed runs |
| 130 | + const errors: string[] = []; |
| 131 | + for (const tr of taskResults) { |
| 132 | + for (const run of tr.runs) { |
| 133 | + if (!run.invocation.success && run.invocation.error) { |
| 134 | + errors.push(run.invocation.error); |
| 135 | + } |
| 136 | + } |
| 137 | + } |
| 138 | + |
| 139 | + // No failures — can't evaluate error quality, give neutral score |
| 140 | + if (errors.length === 0) return 70; |
| 141 | + |
| 142 | + let totalScore = 0; |
| 143 | + |
| 144 | + for (const error of errors) { |
| 145 | + let errorScore = 0; |
| 146 | + |
| 147 | + // Check 1: Error message is not empty |
| 148 | + if (error.length > 0) errorScore += 0.3; |
| 149 | + |
| 150 | + // Check 2: Error message is descriptive (> 10 chars) |
| 151 | + if (error.length > 10) errorScore += 0.3; |
| 152 | + |
| 153 | + // Check 3: Error is structured (not a raw stack trace) |
| 154 | + // Raw stack traces contain "at " lines and file paths — penalize these |
| 155 | + const stackTraceLines = error |
| 156 | + .split("\n") |
| 157 | + .filter((line) => /^\s+at\s/.test(line)).length; |
| 158 | + const totalLines = error.split("\n").length; |
| 159 | + if (totalLines > 0 && stackTraceLines / totalLines < 0.5) { |
| 160 | + errorScore += 0.4; |
| 161 | + } |
| 162 | + |
| 163 | + totalScore += errorScore; |
| 164 | + } |
| 165 | + |
| 166 | + return Math.round((totalScore / errors.length) * 100); |
| 167 | +} |
0 commit comments