feat: Drop raw schema from get-dataset, stop nudging get-dataset-schema

Calibration outcome for #882 (probe: 10 top store Actors; Mixpanel: the
get-dataset-schema tool is rarely called):

- get-dataset no longer returns the raw Apify dataset.schema (93–95% of
  response bytes on top Actors, 23–39% phantom fields). The flat `fields`
  list it already returns is the complete, projection-ready inventory.
- Remove the nudges that steer the LLM into get-dataset-schema (a rarely
  needed, context-heavy call): drop it from get-dataset's description and
  nextStep, and from get-dataset-items' last-page nextStep (now points at
  get-dataset for the field list).
- get-dataset-schema stays as an on-demand tool, unchanged — no depth cap;
  when explicitly called it returns the full inferred schema.

Note for apify-mcp-server-internal: get-dataset structuredContent no longer
carries the schema key.

https://claude.ai/code/session_01Sf9wACoa9h9y2m2WZ2Sde5

Author: claude

src/tools/common/get_dataset.ts

@@ -22,7 +22,7 @@ export const getDataset: ToolEntry = Object.freeze({
     name: HelperTools.DATASET_GET,
     description: dedent`
         Get metadata for a dataset (collection of structured data created by an Actor run).
-        The results will include dataset details such as itemCount, schema, fields, and stats.
+        The results will include dataset details such as itemCount, fields, and stats.
         Use fields to understand structure for filtering with ${HelperTools.DATASET_GET_ITEMS}.
         Note: itemCount updates may be delayed by up to ~5 seconds.
 
@@ -51,14 +51,20 @@ export const getDataset: ToolEntry = Object.freeze({
         if (!dataset) {
             return buildStorageNotFound(`Dataset '${datasetId}' not found.`);
         }
+        // The API also returns a raw `schema` (untyped in apify-client). It is 93–95% of the
+        // response bytes on top store Actors and declares fields that may be absent from the
+        // data, so drop it — get-dataset-schema infers a compact schema from real items (#882).
+        const { schema, ...metadata } = dataset as typeof dataset & { schema?: unknown };
         // Apify returns `fields` slash-separated AND with array indices expanded
         // (e.g. `latestComments/0/owner/username`). For a real Instagram-scraper
         // dataset this inflates ~78 schema fields into 528 paths (~85% bloat) and
         // produces slash-notation paths that aren't directly usable as projection
         // hints for `get-dataset-items` (which expects dot-notation). Run the same
         // normalization `buildRunDataset` applies so this tool's `fields` matches
         // the structured `storages.datasets.default.fields` shape.
-        const normalized = dataset.fields ? { ...dataset, fields: normalizeDatasetFields(dataset.fields) } : dataset;
+        const normalized = metadata.fields
+            ? { ...metadata, fields: normalizeDatasetFields(metadata.fields) }
+            : metadata;
         const fieldCount = Array.isArray(normalized.fields) ? normalized.fields.length : undefined;
         const summary = `Dataset '${normalized.name ?? datasetId}' has ${normalized.itemCount ?? 0} items${fieldCount !== undefined ? `, ${fieldCount} fields` : ''}.`;
         const nextStep = `Use ${HelperTools.DATASET_GET_ITEMS} with datasetId=${datasetId} and limit (for example 20) to fetch items.`;

src/tools/common/storage_helpers.ts

@@ -67,7 +67,7 @@ export function buildStorageListSummaryNextStep(params: {
 
 /**
  * Pagination-aware {summary, nextStep}: when more items remain, point at the next page;
- * otherwise point at get-dataset-schema for structure inspection.
+ * otherwise point at get-dataset for the field list (structure lives there, not in a heavy schema dump).
  */
 export function buildDatasetItemsSummaryNextStep(params: {
     datasetId: string;
@@ -88,7 +88,7 @@ export function buildDatasetItemsSummaryNextStep(params: {
             : `Fetched ${itemCount} of ${totalItemCount} items (offset=${offset}); no more pages.`;
     return {
         summary,
-        nextStep: `Use ${HelperTools.DATASET_SCHEMA_GET} with datasetId=${datasetId} to inspect structure if needed.`,
+        nextStep: `Use ${HelperTools.DATASET_GET} with datasetId=${datasetId} to see the field list if you need the data structure.`,
     };
 }
 

src/tools/structured_output_schemas.ts

@@ -541,7 +541,8 @@ export const datasetItemsOutputSchema = {
 
 /**
  * Schema for dataset metadata (get-dataset). Documents the fields the LLM acts on; the raw API
- * response carries more keys (stats, schema, access settings), allowed as additional properties.
+ * response carries more keys (stats, access settings), allowed as additional properties.
+ * The raw `schema` key is stripped by the tool — get-dataset-schema owns schema output (#882).
  */
 export const datasetMetadataOutputSchema = {
     type: 'object' as const,

tests/unit/tools.get_dataset.test.ts

@@ -54,6 +54,21 @@ describe('get-dataset', () => {
         expect(content[0].text).toContain("Dataset 'missing' not found");
     });
 
+    it('strips the raw schema field from the response', async () => {
+        // Calibration probe (#882): raw `dataset.schema` was 93–95% of the response bytes on
+        // top store Actors and declares fields absent from the data. get-dataset-schema is
+        // the schema source; this tool returns metadata only.
+        const result = await (getDataset as HelperTool).call(
+            stubToolCallContext(
+                { datasetId: 'ds-1' },
+                stubApifyClient({ ...MOCK_DATASET, schema: { fields: {}, views: {} } }),
+            ),
+        );
+        const { structuredContent } = result as { structuredContent: Record<string, unknown> };
+        expect(structuredContent).not.toHaveProperty('schema');
+        expect(structuredContent).toMatchObject(MOCK_DATASET);
+    });
+
     it('rejects empty datasetId via ajv validation', () => {
         const tool = getDataset as HelperTool;
         expect(tool.ajvValidate({ datasetId: '' })).toBe(false);

tests/unit/tools.get_dataset_items.test.ts

@@ -181,7 +181,7 @@ describe('get-dataset-items', () => {
         expect(structuredContent.datasetId).toBe('user~my-dataset');
     });
 
-    it('emits a last-page summary and a schema nextStep when all items are returned', async () => {
+    it('emits a last-page summary and a get-dataset nextStep when all items are returned', async () => {
         const result = await (getDatasetItems as HelperTool).call(
             stubToolCallContext({ datasetId: 'ds-1' }, stubApifyClient()),
         );
@@ -190,7 +190,7 @@ describe('get-dataset-items', () => {
         };
 
         expect(structuredContent.summary).toBe('Fetched all 1 items.');
-        expect(structuredContent.nextStep).toContain(HelperTools.DATASET_SCHEMA_GET);
+        expect(structuredContent.nextStep).toContain(HelperTools.DATASET_GET);
         expect(structuredContent.nextStep).toContain('datasetId=ds-1');
         // summary + nextStep ship as a separate text block after the fenced data.
         expect(content[1].text).toBe(`${structuredContent.summary}\n${structuredContent.nextStep}`);