Skip to content

Commit 67318af

Browse files
github-actions[bot]github-actions[bot]
committed
feat: pagination and streaming for bounded, context-friendly query results
Add offset/limit pagination to data functions so MCP clients get bounded results with metadata to request more, and CLI consumers can process results incrementally via NDJSON. - New src/paginate.js with paginate(), paginateResult(), MCP_DEFAULTS, MCP_MAX_LIMIT utilities - Pagination support in listFunctionsData, queryNameData, whereData, rolesData, listEntryPointsData - Export limiting for DOT/Mermaid (truncation comments) and JSON (edge pagination) - MCP tool schemas updated with limit/offset props and sensible defaults (e.g. list_functions: 100, query_function: 50) - CLI --limit, --offset, --ndjson flags on query, where, roles, flow - Programmatic API exports from index.js - 33 new integration tests covering all pagination scenarios Impact: 19 functions changed, 18 affected
1 parent b97af8c commit 67318af

9 files changed

Lines changed: 578 additions & 15 deletions

File tree

‎src/cli.js‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,8 +100,17 @@ program
100100
.option('-T, --no-tests', 'Exclude test/spec files from results')
101101
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
102102
.option('-j, --json', 'Output as JSON')
103+
.option('--limit <number>', 'Max results to return')
104+
.option('--offset <number>', 'Skip N results (default: 0)')
105+
.option('--ndjson', 'Newline-delimited JSON output')
103106
.action((name, opts) => {
104-
queryName(name, opts.db, { noTests: resolveNoTests(opts), json: opts.json });
107+
queryName(name, opts.db, {
108+
noTests: resolveNoTests(opts),
109+
json: opts.json,
110+
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
111+
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
112+
ndjson: opts.ndjson,
113+
});
105114
});
106115

107116
program
@@ -282,13 +291,23 @@ program
282291
.option('-T, --no-tests', 'Exclude test/spec files from results')
283292
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
284293
.option('-j, --json', 'Output as JSON')
294+
.option('--limit <number>', 'Max results to return')
295+
.option('--offset <number>', 'Skip N results (default: 0)')
296+
.option('--ndjson', 'Newline-delimited JSON output')
285297
.action((name, opts) => {
286298
if (!name && !opts.file) {
287299
console.error('Provide a symbol name or use --file <path>');
288300
process.exit(1);
289301
}
290302
const target = opts.file || name;
291-
where(target, opts.db, { file: !!opts.file, noTests: resolveNoTests(opts), json: opts.json });
303+
where(target, opts.db, {
304+
file: !!opts.file,
305+
noTests: resolveNoTests(opts),
306+
json: opts.json,
307+
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
308+
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
309+
ndjson: opts.ndjson,
310+
});
292311
});
293312

294313
program
@@ -604,6 +623,9 @@ program
604623
.option('-T, --no-tests', 'Exclude test/spec files')
605624
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
606625
.option('-j, --json', 'Output as JSON')
626+
.option('--limit <number>', 'Max results to return')
627+
.option('--offset <number>', 'Skip N results (default: 0)')
628+
.option('--ndjson', 'Newline-delimited JSON output')
607629
.action((opts) => {
608630
if (opts.role && !VALID_ROLES.includes(opts.role)) {
609631
console.error(`Invalid role "${opts.role}". Valid roles: ${VALID_ROLES.join(', ')}`);
@@ -614,6 +636,9 @@ program
614636
file: opts.file,
615637
noTests: resolveNoTests(opts),
616638
json: opts.json,
639+
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
640+
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
641+
ndjson: opts.ndjson,
617642
});
618643
});
619644

@@ -692,6 +717,9 @@ program
692717
.option('-T, --no-tests', 'Exclude test/spec files from results')
693718
.option('--include-tests', 'Include test/spec files (overrides excludeTests config)')
694719
.option('-j, --json', 'Output as JSON')
720+
.option('--limit <number>', 'Max results to return')
721+
.option('--offset <number>', 'Skip N results (default: 0)')
722+
.option('--ndjson', 'Newline-delimited JSON output')
695723
.action(async (name, opts) => {
696724
if (!name && !opts.list) {
697725
console.error('Provide a function/entry point name or use --list to see all entry points.');
@@ -709,6 +737,9 @@ program
709737
kind: opts.kind,
710738
noTests: resolveNoTests(opts),
711739
json: opts.json,
740+
limit: opts.limit ? parseInt(opts.limit, 10) : undefined,
741+
offset: opts.offset ? parseInt(opts.offset, 10) : undefined,
742+
ndjson: opts.ndjson,
712743
});
713744
});
714745

‎src/export.js‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import path from 'node:path';
2+
import { paginateResult } from './paginate.js';
23
import { isTestFile } from './queries.js';
34

45
const DEFAULT_MIN_CONFIDENCE = 0.5;
@@ -10,6 +11,7 @@ export function exportDOT(db, opts = {}) {
1011
const fileLevel = opts.fileLevel !== false;
1112
const noTests = opts.noTests || false;
1213
const minConf = opts.minConfidence ?? DEFAULT_MIN_CONFIDENCE;
14+
const edgeLimit = opts.limit;
1315
const lines = [
1416
'digraph codegraph {',
1517
' rankdir=LR;',
@@ -30,6 +32,8 @@ export function exportDOT(db, opts = {}) {
3032
`)
3133
.all(minConf);
3234
if (noTests) edges = edges.filter((e) => !isTestFile(e.source) && !isTestFile(e.target));
35+
const totalFileEdges = edges.length;
36+
if (edgeLimit && edges.length > edgeLimit) edges = edges.slice(0, edgeLimit);
3337

3438
// Try to use directory nodes from DB (built by structure analysis)
3539
const hasDirectoryNodes =
@@ -95,6 +99,9 @@ export function exportDOT(db, opts = {}) {
9599
for (const { source, target } of edges) {
96100
lines.push(` "${source}" -> "${target}";`);
97101
}
102+
if (edgeLimit && totalFileEdges > edgeLimit) {
103+
lines.push(` // Truncated: showing ${edges.length} of ${totalFileEdges} edges`);
104+
}
98105
} else {
99106
let edges = db
100107
.prepare(`
@@ -111,6 +118,8 @@ export function exportDOT(db, opts = {}) {
111118
.all(minConf);
112119
if (noTests)
113120
edges = edges.filter((e) => !isTestFile(e.source_file) && !isTestFile(e.target_file));
121+
const totalFnEdges = edges.length;
122+
if (edgeLimit && edges.length > edgeLimit) edges = edges.slice(0, edgeLimit);
114123

115124
for (const e of edges) {
116125
const sId = `${e.source_file}:${e.source_name}`.replace(/[^a-zA-Z0-9_]/g, '_');
@@ -119,6 +128,9 @@ export function exportDOT(db, opts = {}) {
119128
lines.push(` ${tId} [label="${e.target_name}\\n${path.basename(e.target_file)}"];`);
120129
lines.push(` ${sId} -> ${tId};`);
121130
}
131+
if (edgeLimit && totalFnEdges > edgeLimit) {
132+
lines.push(` // Truncated: showing ${edges.length} of ${totalFnEdges} edges`);
133+
}
122134
}
123135

124136
lines.push('}');
@@ -169,6 +181,7 @@ export function exportMermaid(db, opts = {}) {
169181
const noTests = opts.noTests || false;
170182
const minConf = opts.minConfidence ?? DEFAULT_MIN_CONFIDENCE;
171183
const direction = opts.direction || 'LR';
184+
const edgeLimit = opts.limit;
172185
const lines = [`flowchart ${direction}`];
173186

174187
let nodeCounter = 0;
@@ -190,6 +203,8 @@ export function exportMermaid(db, opts = {}) {
190203
`)
191204
.all(minConf);
192205
if (noTests) edges = edges.filter((e) => !isTestFile(e.source) && !isTestFile(e.target));
206+
const totalMermaidFileEdges = edges.length;
207+
if (edgeLimit && edges.length > edgeLimit) edges = edges.slice(0, edgeLimit);
193208

194209
// Collect all files referenced in edges
195210
const allFiles = new Set();
@@ -248,6 +263,9 @@ export function exportMermaid(db, opts = {}) {
248263
for (const { source, target, labels } of edgeMap.values()) {
249264
lines.push(` ${nodeId(source)} -->|${[...labels].join(', ')}| ${nodeId(target)}`);
250265
}
266+
if (edgeLimit && totalMermaidFileEdges > edgeLimit) {
267+
lines.push(` %% Truncated: showing ${edges.length} of ${totalMermaidFileEdges} edges`);
268+
}
251269
} else {
252270
let edges = db
253271
.prepare(`
@@ -265,6 +283,8 @@ export function exportMermaid(db, opts = {}) {
265283
.all(minConf);
266284
if (noTests)
267285
edges = edges.filter((e) => !isTestFile(e.source_file) && !isTestFile(e.target_file));
286+
const totalMermaidFnEdges = edges.length;
287+
if (edgeLimit && edges.length > edgeLimit) edges = edges.slice(0, edgeLimit);
268288

269289
// Group nodes by file for subgraphs
270290
const fileNodes = new Map();
@@ -301,6 +321,9 @@ export function exportMermaid(db, opts = {}) {
301321
const tId = nodeId(`${e.target_file}::${e.target_name}`);
302322
lines.push(` ${sId} -->|${e.edge_kind}| ${tId}`);
303323
}
324+
if (edgeLimit && totalMermaidFnEdges > edgeLimit) {
325+
lines.push(` %% Truncated: showing ${edges.length} of ${totalMermaidFnEdges} edges`);
326+
}
304327

305328
// Role styling — query roles for all referenced nodes
306329
const allKeys = [...nodeIdMap.keys()];
@@ -348,5 +371,6 @@ export function exportJSON(db, opts = {}) {
348371
.all(minConf);
349372
if (noTests) edges = edges.filter((e) => !isTestFile(e.source) && !isTestFile(e.target));
350373

351-
return { nodes, edges };
374+
const base = { nodes, edges };
375+
return paginateResult(base, 'edges', { limit: opts.limit, offset: opts.offset });
352376
}

‎src/flow.js‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
*/
77

88
import { openReadonlyOrFail } from './db.js';
9+
import { paginateResult } from './paginate.js';
910
import { isTestFile, kindIcon } from './queries.js';
1011
import { FRAMEWORK_ENTRY_PREFIXES } from './structure.js';
1112

@@ -69,7 +70,8 @@ export function listEntryPointsData(dbPath, opts = {}) {
6970
}
7071

7172
db.close();
72-
return { entries, byType, count: entries.length };
73+
const base = { entries, byType, count: entries.length };
74+
return paginateResult(base, 'entries', { limit: opts.limit, offset: opts.offset });
7375
}
7476

7577
/**
@@ -285,7 +287,16 @@ function findBestMatch(db, name, opts = {}) {
285287
*/
286288
export function flow(name, dbPath, opts = {}) {
287289
if (opts.list) {
288-
const data = listEntryPointsData(dbPath, { noTests: opts.noTests });
290+
const data = listEntryPointsData(dbPath, {
291+
noTests: opts.noTests,
292+
limit: opts.limit,
293+
offset: opts.offset,
294+
});
295+
if (opts.ndjson) {
296+
if (data._pagination) console.log(JSON.stringify({ _meta: data._pagination }));
297+
for (const e of data.entries) console.log(JSON.stringify(e));
298+
return;
299+
}
289300
if (opts.json) {
290301
console.log(JSON.stringify(data, null, 2));
291302
return;

‎src/index.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ export { setVerbose } from './logger.js';
7070
export { manifesto, manifestoData, RULE_DEFS } from './manifesto.js';
7171
// Native engine
7272
export { isNativeAvailable } from './native.js';
73+
// Pagination utilities
74+
export { MCP_DEFAULTS, MCP_MAX_LIMIT, paginate, paginateResult } from './paginate.js';
7375

7476
// Unified parser API
7577
export { getActiveEngine, parseFileAuto, parseFilesAuto } from './parser.js';

‎src/mcp.js‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import { createRequire } from 'node:module';
99
import { findCycles } from './cycles.js';
1010
import { findDbPath } from './db.js';
11+
import { MCP_DEFAULTS, MCP_MAX_LIMIT } from './paginate.js';
1112
import { ALL_SYMBOL_KINDS, diffImpactMermaid, VALID_ROLES } from './queries.js';
1213

1314
const REPO_PROP = {
@@ -17,6 +18,11 @@ const REPO_PROP = {
1718
},
1819
};
1920

21+
const PAGINATION_PROPS = {
22+
limit: { type: 'number', description: 'Max results to return (pagination)' },
23+
offset: { type: 'number', description: 'Skip this many results (pagination, default: 0)' },
24+
};
25+
2026
const BASE_TOOLS = [
2127
{
2228
name: 'query_function',
@@ -31,6 +37,7 @@ const BASE_TOOLS = [
3137
default: 2,
3238
},
3339
no_tests: { type: 'boolean', description: 'Exclude test files', default: false },
40+
...PAGINATION_PROPS,
3441
},
3542
required: ['name'],
3643
},
@@ -214,6 +221,7 @@ const BASE_TOOLS = [
214221
default: false,
215222
},
216223
no_tests: { type: 'boolean', description: 'Exclude test files', default: false },
224+
...PAGINATION_PROPS,
217225
},
218226
required: ['target'],
219227
},
@@ -266,6 +274,7 @@ const BASE_TOOLS = [
266274
description: 'File-level graph (true) or function-level (false)',
267275
default: true,
268276
},
277+
...PAGINATION_PROPS,
269278
},
270279
required: ['format'],
271280
},
@@ -280,6 +289,7 @@ const BASE_TOOLS = [
280289
file: { type: 'string', description: 'Filter by file path (partial match)' },
281290
pattern: { type: 'string', description: 'Filter by function name (partial match)' },
282291
no_tests: { type: 'boolean', description: 'Exclude test files', default: false },
292+
...PAGINATION_PROPS,
283293
},
284294
},
285295
},
@@ -319,6 +329,7 @@ const BASE_TOOLS = [
319329
},
320330
file: { type: 'string', description: 'Scope to a specific file (partial match)' },
321331
no_tests: { type: 'boolean', description: 'Exclude test files', default: false },
332+
...PAGINATION_PROPS,
322333
},
323334
},
324335
},
@@ -400,6 +411,7 @@ const BASE_TOOLS = [
400411
type: 'object',
401412
properties: {
402413
no_tests: { type: 'boolean', description: 'Exclude test files', default: false },
414+
...PAGINATION_PROPS,
403415
},
404416
},
405417
},
@@ -604,7 +616,11 @@ export async function startMCPServer(customDbPath, options = {}) {
604616
let result;
605617
switch (name) {
606618
case 'query_function':
607-
result = queryNameData(args.name, dbPath, { noTests: args.no_tests });
619+
result = queryNameData(args.name, dbPath, {
620+
noTests: args.no_tests,
621+
limit: Math.min(args.limit ?? MCP_DEFAULTS.query_function, MCP_MAX_LIMIT),
622+
offset: args.offset ?? 0,
623+
});
608624
break;
609625
case 'file_deps':
610626
result = fileDepsData(args.file, dbPath, { noTests: args.no_tests });
@@ -666,6 +682,8 @@ export async function startMCPServer(customDbPath, options = {}) {
666682
result = whereData(args.target, dbPath, {
667683
file: args.file_mode,
668684
noTests: args.no_tests,
685+
limit: Math.min(args.limit ?? MCP_DEFAULTS.where, MCP_MAX_LIMIT),
686+
offset: args.offset ?? 0,
669687
});
670688
break;
671689
case 'diff_impact':
@@ -705,15 +723,21 @@ export async function startMCPServer(customDbPath, options = {}) {
705723
const { exportDOT, exportMermaid, exportJSON } = await import('./export.js');
706724
const db = new Database(findDbPath(dbPath), { readonly: true });
707725
const fileLevel = args.file_level !== false;
726+
const exportLimit = args.limit
727+
? Math.min(args.limit, MCP_MAX_LIMIT)
728+
: MCP_DEFAULTS.export_graph;
708729
switch (args.format) {
709730
case 'dot':
710-
result = exportDOT(db, { fileLevel });
731+
result = exportDOT(db, { fileLevel, limit: exportLimit });
711732
break;
712733
case 'mermaid':
713-
result = exportMermaid(db, { fileLevel });
734+
result = exportMermaid(db, { fileLevel, limit: exportLimit });
714735
break;
715736
case 'json':
716-
result = exportJSON(db);
737+
result = exportJSON(db, {
738+
limit: exportLimit,
739+
offset: args.offset ?? 0,
740+
});
717741
break;
718742
default:
719743
db.close();
@@ -735,13 +759,17 @@ export async function startMCPServer(customDbPath, options = {}) {
735759
file: args.file,
736760
pattern: args.pattern,
737761
noTests: args.no_tests,
762+
limit: Math.min(args.limit ?? MCP_DEFAULTS.list_functions, MCP_MAX_LIMIT),
763+
offset: args.offset ?? 0,
738764
});
739765
break;
740766
case 'node_roles':
741767
result = rolesData(dbPath, {
742768
role: args.role,
743769
file: args.file,
744770
noTests: args.no_tests,
771+
limit: Math.min(args.limit ?? MCP_DEFAULTS.node_roles, MCP_MAX_LIMIT),
772+
offset: args.offset ?? 0,
745773
});
746774
break;
747775
case 'structure': {
@@ -793,6 +821,8 @@ export async function startMCPServer(customDbPath, options = {}) {
793821
const { listEntryPointsData } = await import('./flow.js');
794822
result = listEntryPointsData(dbPath, {
795823
noTests: args.no_tests,
824+
limit: Math.min(args.limit ?? MCP_DEFAULTS.list_entry_points, MCP_MAX_LIMIT),
825+
offset: args.offset ?? 0,
796826
});
797827
break;
798828
}

0 commit comments

Comments
 (0)