feat: add content flattener for improved token management

mcp-server/src/helpers/content-flattener.ts

@@ -0,0 +1,106 @@
+/**
+ * Content flattening utilities for improved token management.
+ *
+ * LLM context windows have limited token capacity. These utilities
+ * reduce token usage by:
+ * - Minifying JSON output (removing unnecessary whitespace)
+ * - Stripping fields that don't add value for LLM reasoning
+ * - Truncating large result sets with a summary
+ *
+ * @see https://github.com/uscensusbureau/us-census-bureau-data-api-mcp/issues/70
+ */
+
+/**
+ * Fields commonly returned by the Census API or database that are not
+ * useful for LLM reasoning and can be safely stripped to save tokens.
+ */
+const DEFAULT_STRIP_FIELDS = new Set([
+  'created_at',
+  'updated_at',
+  'parent_geography_level_id',
+])
+
+/**
+ * Serialize data to compact JSON, optionally stripping unnecessary fields.
+ *
+ * Unlike JSON.stringify(data, null, 2), this produces minimal output
+ * with no extra whitespace, and filters out fields that waste tokens.
+ */
+export function flattenJson(
+  data: unknown,
+  options?: {
+    stripFields?: Set<string>
+    maxItems?: number
+  }
+): string {
+  const stripFields = options?.stripFields ?? DEFAULT_STRIP_FIELDS
+  const maxItems = options?.maxItems
+
+  const processed = stripUnusedFields(data, stripFields)
+  const truncated = truncateArray(processed, maxItems)
+
+  return JSON.stringify(truncated)
+}
+
+/**
+ * Build a compact text response with optional result count summary.
+ * Replaces the pattern: `"Found N results:\n\n" + JSON.stringify(data, null, 2)`
+ */
+export function flattenResponse(
+  prefix: string,
+  data: unknown,
+  options?: {
+    stripFields?: Set<string>
+    maxItems?: number
+  }
+): string {
+  const maxItems = options?.maxItems
+  const json = flattenJson(data, options)
+  const itemCount = Array.isArray(data) ? data.length : undefined
+
+  let result = prefix
+
+  if (maxItems && itemCount && itemCount > maxItems) {
+    result += ` (showing ${maxItems} of ${itemCount})`
+  }
+
+  result += '\n' + json
+
+  return result
+}
+
+function stripUnusedFields(data: unknown, fields: Set<string>): unknown {
+  if (data === null || data === undefined) {
+    return data
+  }
+
+  if (Array.isArray(data)) {
+    return data.map((item) => stripUnusedFields(item, fields))
+  }
+
+  if (typeof data === 'object') {
+    const result: Record<string, unknown> = {}
+
+    for (const [key, value] of Object.entries(data as Record<string, unknown>)) {
+      if (!fields.has(key) && value !== null) {
+        result[key] = stripUnusedFields(value, fields)
+      }
+    }
+
+    return result
+  }
+
+  return data
+}
+
+function truncateArray(data: unknown, maxItems?: number): unknown {
+  if (!maxItems || !Array.isArray(data)) {
+    return data
+  }
+
+  if (data.length <= maxItems) {
+    return data
+  }
+
+  return data.slice(0, maxItems)
+}

mcp-server/src/tools/fetch-dataset-geography.tool.ts

@@ -1,5 +1,6 @@
 import { Tool } from '@modelcontextprotocol/sdk/types.js'
 
+import { flattenResponse } from '../helpers/content-flattener.js'
 import { BaseTool } from './base.tool.js'
 import { DatabaseService } from '../services/database.service.js'
 import {
@@ -220,7 +221,10 @@ export class FetchDatasetGeographyTool extends BaseTool<FetchDatasetGeographyArg
             content: [
               {
                 type: 'text',
-                text: `Available geographies for ${args.dataset}${args.year ? ` (${args.year})` : ''}:\n\n${JSON.stringify(parsedGeographyData, null, 2)}`,
+                text: flattenResponse(
+                  `Available geographies for ${args.dataset}${args.year ? ` (${args.year})` : ''}:`,
+                  parsedGeographyData
+                ),
               },
             ],
           }

mcp-server/src/tools/resolve-geography-fips.tool.ts

@@ -1,5 +1,6 @@
 import { Tool } from '@modelcontextprotocol/sdk/types.js'
 
+import { flattenResponse } from '../helpers/content-flattener.js'
 import { BaseTool } from './base.tool.js'
 import { DatabaseService } from '../services/database.service.js'
 import {
@@ -103,7 +104,10 @@ export class ResolveGeographyFipsTool extends BaseTool<ResolveGeographyFipsArgs>
           content: [
             {
               type: 'text',
-              text: `Found ${result.length} Matching Geographies:\n\n${JSON.stringify(result, null, 2)}`,
+              text: flattenResponse(
+                `Found ${result.length} Matching Geographies:`,
+                result
+              ),
             },
           ],
         }

mcp-server/src/tools/search-data-tables.tool.ts

@@ -1,5 +1,6 @@
 import { Tool } from '@modelcontextprotocol/sdk/types.js'
 
+import { flattenResponse } from '../helpers/content-flattener.js'
 import { BaseTool } from './base.tool.js'
 import { DatabaseService } from '../services/database.service.js'
 import {
@@ -82,7 +83,10 @@ export class SearchDataTablesTool extends BaseTool<SearchDataTablesArgs> {
           content: [
             {
               type: 'text',
-              text: `Found ${results.length} Matching Data Table${results.length === 1 ? '' : 's'}:\n\n${JSON.stringify(results, null, 2)}`,
+              text: flattenResponse(
+                `Found ${results.length} Matching Data Table${results.length === 1 ? '' : 's'}:`,
+                results
+              ),
             },
           ],
         }