refactor: prune screenshot diff guidance output

Author: thymikee

skills/agent-device/references/verification.md

@@ -59,7 +59,7 @@ agent-device diff screenshot --baseline ./baseline.png --out /tmp/diff.png --ove
 
 - Text output includes ranked changed regions with screen-space rectangles, shape, size, density, average color, and luminance. JSON also includes normalized bounds.
 - The diff PNG uses a light grayscale current-screen context with changed pixels tinted red and changed regions outlined.
-- Install `tesseract` when you want `diff screenshot` to add best-effort OCR text deltas, movement clusters, added/removed text candidates, and bbox size-change hints. OCR improves the text/JSON descriptions only; it does not change the pixel comparison or the diff PNG.
+- Install `tesseract` when you want `diff screenshot` to add best-effort OCR text deltas, movement clusters, and bbox size-change hints. OCR improves the text/JSON descriptions only; it does not change the pixel comparison or the diff PNG.
 - When OCR is available, `diff screenshot` also reports best-effort non-text visual deltas by masking OCR text boxes out of the pixel diff and clustering the remaining residuals. Treat these as hints for icons, controls, and separators, not semantic icon recognition.
 - Add `--overlay-refs` to `diff screenshot` when you also want a separate current-screen overlay guide. The raw screenshot is still used for pixel comparison; the overlay guide is only context for non-text controls, icons, and tappable regions. When overlay refs intersect changed regions, the output lists the best current-screen ref matches under the affected region.
 

src/utils/__tests__/output.test.ts

@@ -706,27 +706,11 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color'
             possibleTextMetricMismatch: true,
           },
         ],
-        addedText: [
-          {
-            text: 'Accessibility',
-            rect: { x: 220, y: 2520, width: 220, height: 40 },
-            confidence: 93,
-          },
-        ],
-        removedText: [
-          {
-            text: 'VPN',
-            rect: { x: 120, y: 2100, width: 90, height: 36 },
-            confidence: 96,
-          },
-        ],
         movementClusters: [
           {
             texts: ['Wi-Fi', 'Bluetooth'],
-            averageDelta: { x: 10, y: 12 },
             xRange: { min: 10, max: 12 },
             yRange: { min: 10, max: 14 },
-            confidence: 90,
           },
         ],
       },
@@ -739,6 +723,13 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color'
           rect: { x: 80, y: 318, width: 30, height: 30 },
           nearestText: 'Wi-Fi',
         },
+        {
+          index: 2,
+          regionIndex: 1,
+          slot: 'separator',
+          likelyKind: 'separator',
+          rect: { x: 90, y: 360, width: 120, height: 2 },
+        },
       ],
     }),
   );
@@ -752,10 +743,8 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color'
     text,
     /text movement cluster: "Wi-Fi", "Bluetooth" dx=\+10\.\.\+12px dy=\+10\.\.\+14px/,
   );
-  assert.match(text, /added text candidates: "Accessibility" at x=220,y=2520/);
-  assert.match(text, /removed text candidates: "VPN" at x=120,y=2100/);
-  assert.match(text, /non-text controls\/boundaries: icon near "Wi-Fi" r1/);
-  assert.match(text, /largest changed region: r1 top-left 70% of diff, brighter/);
+  assert.match(text, /non-text controls: icon near "Wi-Fi" r1/);
+  assert.match(text, /non-text boundaries: separator r1/);
   assert.match(text, /Changed regions:/);
   assert.match(text, /1\. top-left x=10 y=20 100x40, 70% of diff, change=brighter/);
   assert.match(
@@ -775,9 +764,10 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color'
     text,
     /1 \| "Wi-Fi" \| \+10,\+12 \| \+10,0 \| x=120,y=320,w=60,h=22 \| x=130,y=332,w=70,h=22 \| 94 \| ocr-bbox-size-change/,
   );
-  assert.match(text, /Non-text visual deltas \(showing 1\/1; px\):/);
+  assert.match(text, /Non-text visual deltas \(showing 2\/2; px\):/);
   assert.match(text, /item \| region \| slot \| kind \| bboxCurrent \| nearestText/);
   assert.match(text, /1 \| r1 \| leading \| icon \| x=80,y=318,w=30,h=30 \| "Wi-Fi"/);
+  assert.match(text, /2 \| r1 \| separator \| separator \| x=90,y=360,w=120,h=2 \| -/);
   assert.equal(text.includes('\x1b['), false);
 });
 

src/utils/__tests__/screenshot-diff-non-text.test.ts

@@ -47,15 +47,15 @@ test('summarizeNonTextDiffDeltas masks OCR text and reports leading icon residua
       provider: 'tesseract',
       baselineBlocks: 1,
       currentBlocks: 1,
-      baselineBlocksRaw: [
+      baselineBlocksRaw: [],
+      currentBlocksRaw: [
         {
           text: 'Wi-Fi',
           confidence: 90,
           rect: { x: 68, y: 28, width: 60, height: 24 },
           normalizedRect: { x: 30.91, y: 23.33, width: 27.27, height: 20 },
         },
       ],
-      currentBlocksRaw: [],
       matches: [],
     },
   });
@@ -68,6 +68,40 @@ test('summarizeNonTextDiffDeltas masks OCR text and reports leading icon residua
   assert.equal(deltas[0]?.nearestText, 'Wi-Fi');
 });
 
+test('summarizeNonTextDiffDeltas uses overlapping baseline text when current OCR misses a row', () => {
+  const width = 220;
+  const height = 120;
+  const diffMask = new Uint8Array(width * height);
+  paintMaskRect(diffMask, width, { x: 20, y: 30, width: 20, height: 20 });
+
+  const deltas = summarizeNonTextDiffDeltas({
+    diffMask,
+    width,
+    height,
+    regions: [],
+    ocr: {
+      provider: 'tesseract',
+      baselineBlocks: 1,
+      currentBlocks: 0,
+      baselineBlocksRaw: [
+        {
+          text: 'Wi-Fi',
+          confidence: 90,
+          rect: { x: 68, y: 28, width: 60, height: 24 },
+          normalizedRect: { x: 30.91, y: 23.33, width: 27.27, height: 20 },
+        },
+      ],
+      currentBlocksRaw: [],
+      matches: [],
+    },
+  });
+
+  assert.equal(deltas.length, 1);
+  assert.equal(deltas[0]?.slot, 'leading');
+  assert.equal(deltas[0]?.likelyKind, 'icon');
+  assert.equal(deltas[0]?.nearestText, 'Wi-Fi');
+});
+
 test('summarizeNonTextDiffDeltas omits broad background residuals', () => {
   const width = 220;
   const height = 120;

src/utils/__tests__/screenshot-diff-ocr.test.ts

@@ -8,7 +8,6 @@ import {
   parseTesseractTsv,
   summarizeScreenshotOcr,
   summarizeOcrMovementClusters,
-  summarizeOcrTextChanges,
 } from '../screenshot-diff-ocr.ts';
 
 test('parseTesseractTsv groups word rows into text line blocks', () => {
@@ -71,67 +70,6 @@ test('matchOcrBlocks reports movement and OCR bbox size change', () => {
   assert.equal(matches[0]?.possibleTextMetricMismatch, true);
 });
 
-test('summarizeOcrTextChanges normalizes noisy OCR labels before reporting candidates', () => {
-  const changes = summarizeOcrTextChanges(
-    [
-      {
-        text: 'Airplane Mode',
-        confidence: 95,
-        rect: { x: 100, y: 200, width: 80, height: 20 },
-        normalizedRect: { x: 25, y: 25, width: 20, height: 2.5 },
-      },
-      {
-        text: '2) Personal Hotspot',
-        confidence: 90,
-        rect: { x: 100, y: 260, width: 160, height: 20 },
-        normalizedRect: { x: 25, y: 32.5, width: 40, height: 2.5 },
-      },
-      {
-        text: 'Removed Row',
-        confidence: 91,
-        rect: { x: 100, y: 320, width: 140, height: 20 },
-        normalizedRect: { x: 25, y: 40, width: 35, height: 2.5 },
-      },
-      {
-        text: '4:44',
-        confidence: 96,
-        rect: { x: 10, y: 20, width: 60, height: 20 },
-        normalizedRect: { x: 2.5, y: 2.5, width: 15, height: 2.5 },
-      },
-    ],
-    [
-      {
-        text: 'Airplane Mode @e~',
-        confidence: 91,
-        rect: { x: 120, y: 210, width: 120, height: 20 },
-        normalizedRect: { x: 30, y: 26.25, width: 30, height: 2.5 },
-      },
-      {
-        text: 'Personal Hotspot',
-        confidence: 92,
-        rect: { x: 120, y: 270, width: 140, height: 20 },
-        normalizedRect: { x: 30, y: 33.75, width: 35, height: 2.5 },
-      },
-      {
-        text: 'Added Row',
-        confidence: 93,
-        rect: { x: 120, y: 340, width: 110, height: 20 },
-        normalizedRect: { x: 30, y: 42.5, width: 27.5, height: 2.5 },
-      },
-    ],
-    800,
-  );
-
-  assert.deepEqual(
-    changes.addedText.map((change) => change.text),
-    ['Added Row'],
-  );
-  assert.deepEqual(
-    changes.removedText.map((change) => change.text),
-    ['Removed Row'],
-  );
-});
-
 test('summarizeOcrMovementClusters groups repeated x-axis text movement', () => {
   const clusters = summarizeOcrMovementClusters([
     {

src/utils/output.ts

@@ -336,43 +336,23 @@ function formatScreenshotDiffHints(data: ScreenshotDiffResult): string[] {
     );
   }
 
-  const addedText = data.ocr?.addedText ?? [];
-  if (addedText.length > 0) {
-    hints.push(`added text candidates: ${formatTextChanges(addedText)}`);
-  }
-
-  const removedText = data.ocr?.removedText ?? [];
-  if (removedText.length > 0) {
-    hints.push(`removed text candidates: ${formatTextChanges(removedText)}`);
-  }
-
   const controlDeltas = (data.nonTextDeltas ?? [])
-    .filter((delta) => ['icon', 'toggle', 'chevron', 'separator'].includes(delta.likelyKind))
+    .filter((delta) => ['icon', 'toggle', 'chevron'].includes(delta.likelyKind))
     .slice(0, 3);
   if (controlDeltas.length > 0) {
-    hints.push(`non-text controls/boundaries: ${controlDeltas.map(formatNonTextHint).join('; ')}`);
+    hints.push(`non-text controls: ${controlDeltas.map(formatNonTextHint).join('; ')}`);
   }
 
-  const largestRegion = data.regions?.[0];
-  if (largestRegion) {
-    hints.push(
-      `largest changed region: r${largestRegion.index} ${largestRegion.location} ` +
-        `${largestRegion.shareOfDiffPercentage}% of diff, ${largestRegion.dominantChange}`,
-    );
+  const boundaryDeltas = (data.nonTextDeltas ?? [])
+    .filter((delta) => delta.likelyKind === 'separator')
+    .slice(0, 2);
+  if (boundaryDeltas.length > 0) {
+    hints.push(`non-text boundaries: ${boundaryDeltas.map(formatNonTextHint).join('; ')}`);
   }
 
   return hints.slice(0, 6);
 }
 
-function formatTextChanges(
-  changes: Array<{ text: string; rect: { x: number; y: number; width: number; height: number } }>,
-): string {
-  return changes
-    .slice(0, 3)
-    .map((change) => `${JSON.stringify(change.text)} at x=${change.rect.x},y=${change.rect.y}`)
-    .join(', ');
-}
-
 function formatNonTextHint(delta: {
   likelyKind: string;
   nearestText?: string;

src/utils/screenshot-diff-non-text.ts

@@ -84,11 +84,11 @@ export function summarizeNonTextDiffDeltas(params: {
   const rawComponents = findConnectedComponents(maskedDiff, params.width, params.height);
   const mergedComponents = mergeNearbyComponents(rawComponents, MERGE_GAP_PX);
   const currentRows = groupOcrRows(params.ocr?.currentBlocksRaw ?? []);
-  const fallbackTextBlocks = getOcrBlocks(params.ocr);
+  const baselineRows = groupOcrRows(params.ocr?.baselineBlocksRaw ?? []);
   return (
     mergedComponents
       .filter(hasUsefulComponentSize)
-      .map((component) => toNonTextDelta(component, params, currentRows, fallbackTextBlocks))
+      .map((component) => toNonTextDelta(component, params, currentRows, baselineRows))
       // Status bars and top chrome tend to produce noisy residuals around time,
       // signal, and battery text; changed regions still report that area.
       .filter((delta) => delta.rect.y >= params.height * MIN_CONTENT_Y_RATIO)
@@ -197,11 +197,11 @@ function toNonTextDelta(
     regions: ScreenshotDiffRegion[];
   },
   currentRows: OcrRow[],
-  fallbackTextBlocks: ScreenshotOcrBlock[],
+  baselineRows: OcrRow[],
 ): ScoredNonTextDelta {
   const rect = componentToRect(component);
   const regionIndex = findContainingRegionIndex(rect, params.regions);
-  const textAnchor = findTextAnchor(rect, currentRows, fallbackTextBlocks);
+  const textAnchor = findTextAnchor(rect, currentRows, baselineRows);
   const slot = classifySlot(rect, textAnchor?.block.rect, params.width);
   const likelyKind = classifyLikelyKind(rect, slot, component.differentPixels, params);
   const scoreParams = {
@@ -341,11 +341,12 @@ function findContainingRegionIndex(
 function findTextAnchor(
   rect: Rect,
   currentRows: OcrRow[],
-  fallbackTextBlocks: ScreenshotOcrBlock[],
+  baselineRows: OcrRow[],
 ): { block: ScreenshotOcrBlock; distance: number } | undefined {
-  const row = findOverlappingRow(rect, currentRows);
-  if (row) return findNearestText(rect, row.blocks);
-  return findNearestText(rect, fallbackTextBlocks);
+  const currentRow = findOverlappingRow(rect, currentRows);
+  if (currentRow) return findNearestText(rect, currentRow.blocks);
+  const baselineRow = findOverlappingRow(rect, baselineRows);
+  return baselineRow ? findNearestText(rect, baselineRow.blocks) : undefined;
 }
 
 function findOverlappingRow(rect: Rect, rows: OcrRow[]): OcrRow | undefined {
@@ -410,10 +411,6 @@ function unionRects(rects: Rect[]): Rect {
   return { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
 }
 
-function getOcrBlocks(ocr: ScreenshotOcrAnalysis | undefined): ScreenshotOcrBlock[] {
-  return ocr ? [...ocr.baselineBlocksRaw, ...ocr.currentBlocksRaw] : [];
-}
-
 function cleanOcrAnchorText(text: string): string {
   return text
     .trim()