Skip to content

Commit 4c51e5c

Browse files
authored
Merge pull request #3541 from superdoc-dev/andrii/llm-tools-bundles
llm tools presets
2 parents 64d9236 + 09d85ec commit 4c51e5c

15 files changed

Lines changed: 1550 additions & 530 deletions

File tree

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/**
2+
* MCP_PRESET env var selects which LLM-tools preset the server registers.
3+
* Currently only 'legacy' is supported. Unknown preset ids must fail fast at
4+
* startup so misconfiguration is visible instead of silently falling back to
5+
* the default.
6+
*/
7+
8+
import { describe, expect, test } from 'bun:test';
9+
import { spawn } from 'node:child_process';
10+
import path from 'node:path';
11+
12+
const REPO_ROOT = path.resolve(import.meta.dir, '../../../..');
13+
const MCP_ENTRY = path.join(REPO_ROOT, 'apps/mcp/src/server.ts');
14+
15+
type RunResult = { code: number | null; stderr: string };
16+
17+
function runServer(env: NodeJS.ProcessEnv, timeoutMs = 2000): Promise<RunResult> {
18+
return new Promise((resolve) => {
19+
const proc = spawn('bun', ['run', MCP_ENTRY], {
20+
cwd: REPO_ROOT,
21+
env: { ...process.env, ...env },
22+
stdio: ['pipe', 'pipe', 'pipe'],
23+
});
24+
25+
let stderr = '';
26+
proc.stderr.on('data', (chunk) => {
27+
stderr += chunk;
28+
});
29+
30+
// The MCP server runs forever waiting on stdio. We only care about whether
31+
// it exits fast (rejecting bad preset id) or stays alive (accepting preset).
32+
// For the success case we kill after a short window.
33+
const timer = setTimeout(() => {
34+
proc.kill('SIGTERM');
35+
}, timeoutMs);
36+
37+
proc.on('close', (code) => {
38+
clearTimeout(timer);
39+
resolve({ code, stderr });
40+
});
41+
});
42+
}
43+
44+
describe('MCP_PRESET env var', () => {
45+
test('unknown preset id fails fast with exit code 2', async () => {
46+
const result = await runServer({ MCP_PRESET: 'definitely-not-a-preset' });
47+
expect(result.code).toBe(2);
48+
expect(result.stderr).toContain('unknown preset');
49+
expect(result.stderr).toContain('definitely-not-a-preset');
50+
expect(result.stderr).toContain('legacy');
51+
});
52+
53+
test('explicit MCP_PRESET=legacy is accepted (server stays alive)', async () => {
54+
const result = await runServer({ MCP_PRESET: 'legacy' });
55+
// Server should still be running when we kill it (SIGTERM → code is null
56+
// or signal-derived non-2). Either way, it must NOT exit with 2.
57+
expect(result.code).not.toBe(2);
58+
});
59+
});

apps/mcp/src/server.ts

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,18 @@ import { registerAllTools } from './tools/index.js';
99
const require = createRequire(import.meta.url);
1010
const { version } = require('../package.json');
1111

12+
// Validate MCP_PRESET at startup so misconfiguration fails fast instead of
13+
// silently falling back to 'legacy'. Tool registration is wired to legacy via
14+
// the static MCP_TOOL_CATALOG + dispatchIntentTool imports in tools/intent.ts;
15+
// the resolved id is not plumbed further yet. When a non-legacy preset lands,
16+
// pass the id into registerAllTools() so it can route through the registry.
17+
const PRESETS_SUPPORTED = new Set(['legacy']);
18+
const requestedPreset = process.env.MCP_PRESET ?? 'legacy';
19+
if (!PRESETS_SUPPORTED.has(requestedPreset)) {
20+
console.error(`SuperDoc MCP: unknown preset "${requestedPreset}". Supported: ${[...PRESETS_SUPPORTED].join(', ')}.`);
21+
process.exit(2);
22+
}
23+
1224
const server = new McpServer(
1325
{
1426
name: 'superdoc',

packages/sdk/codegen/src/__tests__/cross-lang-parity.test.ts

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ function callPython(command: Record<string, unknown>): Promise<unknown> {
5454

5555
/** Import Node SDK chooseTools (cached). */
5656
let _nodeTools: typeof import('../../../langs/node/src/tools.js') | null = null;
57-
async function nodeTools() {
57+
async function nodeTools(): Promise<typeof import('../../../langs/node/src/tools.js')> {
5858
if (!_nodeTools) {
5959
_nodeTools = await import(path.join(REPO_ROOT, 'packages/sdk/langs/node/src/tools.ts'));
6060
}
@@ -78,6 +78,43 @@ describe('chooseTools parity', () => {
7878
expect(pyResult.meta.toolCount).toBe(nodeResult.meta.toolCount);
7979
expect(nodeResult.meta.toolCount).toBeGreaterThan(0);
8080
});
81+
82+
test('returns same tool count when preset: legacy is explicit (parity)', async () => {
83+
const input = { provider: 'generic' as const, preset: 'legacy' };
84+
85+
const { chooseTools } = await nodeTools();
86+
const nodeResult = await chooseTools(input);
87+
88+
const pyResult = (await callPython({ action: 'chooseTools', input })) as ChooseResult & {
89+
meta: { preset?: string };
90+
};
91+
92+
expect(pyResult.meta.provider).toBe(nodeResult.meta.provider);
93+
expect(pyResult.meta.toolCount).toBe(nodeResult.meta.toolCount);
94+
expect(pyResult.meta.preset).toBe('legacy');
95+
expect(nodeResult.meta.preset).toBe('legacy');
96+
});
97+
});
98+
99+
// --------------------------------------------------------------------------
100+
// Preset registry parity
101+
// --------------------------------------------------------------------------
102+
103+
describe('Preset registry parity', () => {
104+
test('Node and Python expose the same DEFAULT_PRESET and registered ids', async () => {
105+
const { DEFAULT_PRESET: nodeDefault, listPresets: nodeList } = await nodeTools();
106+
const nodePresets = nodeList();
107+
108+
const pyResult = (await callPython({ action: 'listPresets' })) as {
109+
defaultPreset: string;
110+
presets: string[];
111+
};
112+
113+
expect(pyResult.defaultPreset).toBe(nodeDefault);
114+
expect(nodeDefault).toBe('legacy');
115+
// Both runtimes register the same preset set (order-agnostic).
116+
expect([...pyResult.presets].sort()).toEqual([...nodePresets].sort());
117+
});
81118
});
82119

83120
// --------------------------------------------------------------------------
Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
import { describe, expect, test } from 'bun:test';
2+
import {
3+
chooseTools,
4+
DEFAULT_PRESET,
5+
getPreset,
6+
getMcpPrompt,
7+
getSystemPrompt,
8+
getSystemPromptForProvider,
9+
getToolCatalog,
10+
listPresets,
11+
listTools,
12+
} from '../tools.ts';
13+
import { SuperDocCliError } from '../runtime/errors.js';
14+
15+
const PROVIDERS = ['openai', 'anthropic', 'vercel', 'generic'] as const;
16+
17+
describe('preset registry', () => {
18+
test('DEFAULT_PRESET is "legacy"', () => {
19+
expect(DEFAULT_PRESET).toBe('legacy');
20+
});
21+
22+
test('listPresets() includes "legacy"', () => {
23+
const presets = listPresets();
24+
expect(presets).toContain('legacy');
25+
});
26+
27+
test('getPreset() (no arg) returns the legacy preset', () => {
28+
const preset = getPreset();
29+
expect(preset.id).toBe('legacy');
30+
});
31+
32+
test('getPreset("legacy") returns the legacy preset', () => {
33+
const preset = getPreset('legacy');
34+
expect(preset.id).toBe('legacy');
35+
expect(preset.description).toBeDefined();
36+
expect(preset.supportsCacheControl).toBe(true);
37+
});
38+
39+
test('getPreset("nonexistent") throws PRESET_NOT_FOUND', () => {
40+
try {
41+
getPreset('nonexistent-preset');
42+
throw new Error('Expected getPreset to throw.');
43+
} catch (error) {
44+
expect(error).toBeInstanceOf(SuperDocCliError);
45+
const cliError = error as SuperDocCliError;
46+
expect(cliError.code).toBe('PRESET_NOT_FOUND');
47+
expect(cliError.message).toContain('nonexistent-preset');
48+
const details = cliError.details as { id: string; availablePresets: string[] };
49+
expect(details.id).toBe('nonexistent-preset');
50+
expect(details.availablePresets).toContain('legacy');
51+
}
52+
});
53+
54+
test('getPreset("") throws PRESET_NOT_FOUND (empty string is not the default)', () => {
55+
try {
56+
getPreset('');
57+
throw new Error('Expected getPreset("") to throw.');
58+
} catch (error) {
59+
expect(error).toBeInstanceOf(SuperDocCliError);
60+
expect((error as SuperDocCliError).code).toBe('PRESET_NOT_FOUND');
61+
}
62+
});
63+
64+
test('chooseTools({preset: ""}) throws PRESET_NOT_FOUND (cross-lang parity)', async () => {
65+
await expect(chooseTools({ provider: 'openai', preset: '' })).rejects.toMatchObject({
66+
code: 'PRESET_NOT_FOUND',
67+
});
68+
});
69+
});
70+
71+
describe('public ToolCatalog type — structural access', () => {
72+
test('getToolCatalog().tools entries expose typed properties', async () => {
73+
const catalog = await getToolCatalog();
74+
expect(catalog.tools.length).toBeGreaterThan(0);
75+
const first = catalog.tools[0]!;
76+
// These property accesses validate that ToolCatalog.tools is structurally
77+
// typed (ToolCatalogEntry[]) — not unknown[]. Compile failure here means
78+
// the public catalog row type regressed.
79+
expect(typeof first.toolName).toBe('string');
80+
expect(typeof first.description).toBe('string');
81+
expect(typeof first.mutates).toBe('boolean');
82+
expect(Array.isArray(first.operations)).toBe(true);
83+
expect(typeof first.operations[0]?.operationId).toBe('string');
84+
expect(typeof first.operations[0]?.intentAction).toBe('string');
85+
});
86+
});
87+
88+
describe('chooseTools — default preset equivalence', () => {
89+
for (const provider of PROVIDERS) {
90+
test(`omitting preset equals preset: 'legacy' (${provider})`, async () => {
91+
const implicit = await chooseTools({ provider });
92+
const explicit = await chooseTools({ provider, preset: 'legacy' });
93+
// Tools content identical
94+
expect(implicit.tools).toEqual(explicit.tools);
95+
// Same tool count
96+
expect(implicit.meta.toolCount).toBe(explicit.meta.toolCount);
97+
// Same provider, same cache strategy
98+
expect(implicit.meta.provider).toBe(explicit.meta.provider);
99+
expect(implicit.meta.cacheStrategy).toBe(explicit.meta.cacheStrategy);
100+
// Both echo legacy as resolved preset
101+
expect(implicit.meta.preset).toBe('legacy');
102+
expect(explicit.meta.preset).toBe('legacy');
103+
});
104+
}
105+
106+
test(`chooseTools(provider, preset: 'nonexistent') throws PRESET_NOT_FOUND`, async () => {
107+
await expect(chooseTools({ provider: 'openai', preset: 'nonexistent-preset' })).rejects.toMatchObject({
108+
code: 'PRESET_NOT_FOUND',
109+
});
110+
});
111+
112+
test('meta.preset field is included', async () => {
113+
const { meta } = await chooseTools({ provider: 'openai' });
114+
expect(meta.preset).toBe('legacy');
115+
});
116+
});
117+
118+
describe('catalog + listings — default preset equivalence', () => {
119+
test(`getToolCatalog() equals getToolCatalog('legacy')`, async () => {
120+
const implicit = await getToolCatalog();
121+
const explicit = await getToolCatalog('legacy');
122+
expect(implicit).toEqual(explicit);
123+
});
124+
125+
for (const provider of PROVIDERS) {
126+
test(`listTools(${provider}) equals listTools(${provider}, 'legacy')`, async () => {
127+
const implicit = await listTools(provider);
128+
const explicit = await listTools(provider, 'legacy');
129+
expect(implicit).toEqual(explicit);
130+
});
131+
}
132+
133+
test(`getToolCatalog('nonexistent') throws PRESET_NOT_FOUND`, async () => {
134+
await expect(getToolCatalog('nonexistent-preset')).rejects.toMatchObject({
135+
code: 'PRESET_NOT_FOUND',
136+
});
137+
});
138+
});
139+
140+
describe('system prompts — default preset equivalence', () => {
141+
test(`getSystemPrompt() equals getSystemPrompt('legacy')`, async () => {
142+
const implicit = await getSystemPrompt();
143+
const explicit = await getSystemPrompt('legacy');
144+
expect(implicit).toBe(explicit);
145+
});
146+
147+
test(`getMcpPrompt() equals getMcpPrompt('legacy')`, async () => {
148+
const implicit = await getMcpPrompt();
149+
const explicit = await getMcpPrompt('legacy');
150+
expect(implicit).toBe(explicit);
151+
});
152+
153+
test(`getSystemPromptForProvider({provider}) equals preset: 'legacy'`, async () => {
154+
const implicit = await getSystemPromptForProvider({ provider: 'anthropic', cache: true });
155+
const explicit = await getSystemPromptForProvider({
156+
provider: 'anthropic',
157+
preset: 'legacy',
158+
cache: true,
159+
});
160+
expect(implicit).toEqual(explicit);
161+
});
162+
});
163+
164+
describe('legacy preset direct access', () => {
165+
test('getPreset("legacy").getCatalog() matches getToolCatalog()', async () => {
166+
const direct = await getPreset('legacy').getCatalog();
167+
const viaTopLevel = await getToolCatalog();
168+
expect(direct).toEqual(viaTopLevel);
169+
});
170+
171+
for (const provider of PROVIDERS) {
172+
test(`getPreset("legacy").getTools(${provider}) matches chooseTools({provider}).tools`, async () => {
173+
const direct = await getPreset('legacy').getTools(provider);
174+
const viaTopLevel = await chooseTools({ provider });
175+
expect(direct.tools).toEqual(viaTopLevel.tools);
176+
expect(direct.cacheStrategy).toBe(viaTopLevel.meta.cacheStrategy);
177+
});
178+
}
179+
});

packages/sdk/langs/node/src/index.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,11 +253,17 @@ export {
253253
getSystemPromptForProvider,
254254
getToolCatalog,
255255
listTools,
256+
DEFAULT_PRESET,
257+
getPreset,
258+
listPresets,
256259
} from './tools.js';
257260
export type {
258261
AnthropicSystemPrompt,
259262
CacheStrategy,
260263
SystemPromptForProviderResult,
264+
ToolCatalog,
265+
ToolCatalogEntry,
266+
ToolCatalogOperation,
261267
ToolChooserInput,
262268
ToolProvider,
263269
} from './tools.js';

0 commit comments

Comments
 (0)