feat: add compact screenshot diff hints

Author: thymikee

skills/agent-device/references/verification.md

@@ -59,7 +59,7 @@ agent-device diff screenshot --baseline ./baseline.png --out /tmp/diff.png --ove
 
 - Text output includes ranked changed regions with screen-space rectangles, shape, size, density, average color, and luminance. JSON also includes normalized bounds.
 - The diff PNG uses a light grayscale current-screen context with changed pixels tinted red and changed regions outlined.
-- Install `tesseract` when you want `diff screenshot` to add best-effort OCR text deltas such as moved labels and possible text metric mismatches. OCR improves the text/JSON descriptions only; it does not change the pixel comparison or the diff PNG.
+- Install `tesseract` when you want `diff screenshot` to add best-effort OCR text deltas, movement clusters, added/removed text candidates, and bbox size-change hints. OCR improves the text/JSON descriptions only; it does not change the pixel comparison or the diff PNG.
 - When OCR is available, `diff screenshot` also reports best-effort non-text visual deltas by masking OCR text boxes out of the pixel diff and clustering the remaining residuals. Treat these as hints for icons, controls, and separators, not semantic icon recognition.
 - Add `--overlay-refs` to `diff screenshot` when you also want a separate current-screen overlay guide. The raw screenshot is still used for pixel comparison; the overlay guide is only context for non-text controls, icons, and tappable regions. When overlay refs intersect changed regions, the output lists the best current-screen ref matches under the affected region.
 

src/utils/__tests__/output.test.ts

@@ -706,6 +706,29 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color'
             possibleTextMetricMismatch: true,
           },
         ],
+        addedText: [
+          {
+            text: 'Accessibility',
+            rect: { x: 220, y: 2520, width: 220, height: 40 },
+            confidence: 93,
+          },
+        ],
+        removedText: [
+          {
+            text: 'VPN',
+            rect: { x: 120, y: 2100, width: 90, height: 36 },
+            confidence: 96,
+          },
+        ],
+        movementClusters: [
+          {
+            texts: ['Wi-Fi', 'Bluetooth'],
+            averageDelta: { x: 10, y: 12 },
+            xRange: { min: 10, max: 12 },
+            yRange: { min: 10, max: 14 },
+            confidence: 90,
+          },
+        ],
       },
       nonTextDeltas: [
         {
@@ -724,6 +747,15 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color'
   assert.match(text, /Current overlay:/);
   assert.match(text, /diff\.current-overlay\.png \(1 refs\)/);
   assert.match(text, /500 different \/ 10000 total pixels/);
+  assert.match(text, /Hints:/);
+  assert.match(
+    text,
+    /text movement cluster: "Wi-Fi", "Bluetooth" dx=\+10\.\.\+12px dy=\+10\.\.\+14px/,
+  );
+  assert.match(text, /added text candidates: "Accessibility" at x=220,y=2520/);
+  assert.match(text, /removed text candidates: "VPN" at x=120,y=2100/);
+  assert.match(text, /non-text controls\/boundaries: icon near "Wi-Fi" r1/);
+  assert.match(text, /largest changed region: r1 top-left 70% of diff, brighter/);
   assert.match(text, /Changed regions:/);
   assert.match(text, /1\. top-left x=10 y=20 100x40, 70% of diff, change=brighter/);
   assert.match(
@@ -741,7 +773,7 @@ test('formatScreenshotDiffText renders mismatch with pixel counts without color'
   );
   assert.match(
     text,
-    /1 \| "Wi-Fi" \| \+10,\+12 \| \+10,0 \| x=120,y=320,w=60,h=22 \| x=130,y=332,w=70,h=22 \| 94 \| possible-text-metric-mismatch/,
+    /1 \| "Wi-Fi" \| \+10,\+12 \| \+10,0 \| x=120,y=320,w=60,h=22 \| x=130,y=332,w=70,h=22 \| 94 \| ocr-bbox-size-change/,
   );
   assert.match(text, /Non-text visual deltas \(showing 1\/1; px\):/);
   assert.match(text, /item \| region \| slot \| kind \| bboxCurrent \| nearestText/);

src/utils/__tests__/screenshot-diff-ocr.test.ts

@@ -7,6 +7,8 @@ import {
   matchOcrBlocks,
   parseTesseractTsv,
   summarizeScreenshotOcr,
+  summarizeOcrMovementClusters,
+  summarizeOcrTextChanges,
 } from '../screenshot-diff-ocr.ts';
 
 test('parseTesseractTsv groups word rows into text line blocks', () => {
@@ -44,7 +46,7 @@ test('parseTesseractTsv groups word rows into text line blocks', () => {
   });
 });
 
-test('matchOcrBlocks reports movement and possible text metric mismatch', () => {
+test('matchOcrBlocks reports movement and OCR bbox size change', () => {
   const matches = matchOcrBlocks(
     [
       {
@@ -69,6 +71,101 @@ test('matchOcrBlocks reports movement and possible text metric mismatch', () =>
   assert.equal(matches[0]?.possibleTextMetricMismatch, true);
 });
 
+test('summarizeOcrTextChanges normalizes noisy OCR labels before reporting candidates', () => {
+  const changes = summarizeOcrTextChanges(
+    [
+      {
+        text: 'Airplane Mode',
+        confidence: 95,
+        rect: { x: 100, y: 200, width: 80, height: 20 },
+        normalizedRect: { x: 25, y: 25, width: 20, height: 2.5 },
+      },
+      {
+        text: '2) Personal Hotspot',
+        confidence: 90,
+        rect: { x: 100, y: 260, width: 160, height: 20 },
+        normalizedRect: { x: 25, y: 32.5, width: 40, height: 2.5 },
+      },
+      {
+        text: 'Removed Row',
+        confidence: 91,
+        rect: { x: 100, y: 320, width: 140, height: 20 },
+        normalizedRect: { x: 25, y: 40, width: 35, height: 2.5 },
+      },
+      {
+        text: '4:44',
+        confidence: 96,
+        rect: { x: 10, y: 20, width: 60, height: 20 },
+        normalizedRect: { x: 2.5, y: 2.5, width: 15, height: 2.5 },
+      },
+    ],
+    [
+      {
+        text: 'Airplane Mode @e~',
+        confidence: 91,
+        rect: { x: 120, y: 210, width: 120, height: 20 },
+        normalizedRect: { x: 30, y: 26.25, width: 30, height: 2.5 },
+      },
+      {
+        text: 'Personal Hotspot',
+        confidence: 92,
+        rect: { x: 120, y: 270, width: 140, height: 20 },
+        normalizedRect: { x: 30, y: 33.75, width: 35, height: 2.5 },
+      },
+      {
+        text: 'Added Row',
+        confidence: 93,
+        rect: { x: 120, y: 340, width: 110, height: 20 },
+        normalizedRect: { x: 30, y: 42.5, width: 27.5, height: 2.5 },
+      },
+    ],
+    800,
+  );
+
+  assert.deepEqual(
+    changes.addedText.map((change) => change.text),
+    ['Added Row'],
+  );
+  assert.deepEqual(
+    changes.removedText.map((change) => change.text),
+    ['Removed Row'],
+  );
+});
+
+test('summarizeOcrMovementClusters groups repeated x-axis text movement', () => {
+  const clusters = summarizeOcrMovementClusters([
+    {
+      text: 'Wi-Fi',
+      baselineRect: { x: 100, y: 200, width: 50, height: 20 },
+      currentRect: { x: 286, y: 120, width: 50, height: 20 },
+      delta: { x: 186, y: -80, width: 0, height: 0 },
+      confidence: 96,
+      possibleTextMetricMismatch: false,
+    },
+    {
+      text: 'Bluetooth',
+      baselineRect: { x: 100, y: 260, width: 90, height: 20 },
+      currentRect: { x: 284, y: 190, width: 90, height: 20 },
+      delta: { x: 184, y: -70, width: 0, height: 0 },
+      confidence: 90,
+      possibleTextMetricMismatch: false,
+    },
+    {
+      text: 'Search',
+      baselineRect: { x: 100, y: 500, width: 90, height: 20 },
+      currentRect: { x: 52, y: 560, width: 90, height: 20 },
+      delta: { x: -48, y: 60, width: 0, height: 0 },
+      confidence: 94,
+      possibleTextMetricMismatch: false,
+    },
+  ]);
+
+  assert.equal(clusters.length, 1);
+  assert.deepEqual(clusters[0]?.texts, ['Wi-Fi', 'Bluetooth']);
+  assert.deepEqual(clusters[0]?.xRange, { min: 184, max: 186 });
+  assert.deepEqual(clusters[0]?.yRange, { min: -80, max: -70 });
+});
+
 test('summarizeScreenshotOcr returns undefined when tesseract exits non-zero', async () => {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ocr-test-'));
   const binDir = path.join(dir, 'bin');

src/utils/output.ts

@@ -242,6 +242,12 @@ export function formatScreenshotDiffText(data: ScreenshotDiffResult): string {
     lines.push(`  ${diffCount} different / ${totalPixels} total pixels`);
   }
 
+  const hints = !match && !dimensionMismatch ? formatScreenshotDiffHints(data) : [];
+  if (hints.length > 0) {
+    lines.push('  Hints:');
+    for (const hint of hints) lines.push(`    - ${hint}`);
+  }
+
   const regions = Array.isArray(data.regions) ? data.regions : [];
   if (!match && !dimensionMismatch && regions.length > 0) {
     lines.push('  Changed regions:');
@@ -288,7 +294,7 @@ export function formatScreenshotDiffText(data: ScreenshotDiffResult): string {
           `${formatSignedPixels(delta.width)},${formatSignedPixels(delta.height)} | ` +
           `${formatRect(ocrMatch.baselineRect)} | ${formatRect(ocrMatch.currentRect)} | ` +
           `${ocrMatch.confidence} | ` +
-          `${ocrMatch.possibleTextMetricMismatch ? 'possible-text-metric-mismatch' : '-'}`,
+          `${ocrMatch.possibleTextMetricMismatch ? 'ocr-bbox-size-change' : '-'}`,
       );
     }
   }
@@ -320,6 +326,75 @@ function formatSignedPixels(value: number): string {
   return value > 0 ? `+${value}` : String(value);
 }
 
+function formatScreenshotDiffHints(data: ScreenshotDiffResult): string[] {
+  const hints: string[] = [];
+  const clusters = data.ocr?.movementClusters ?? [];
+  for (const cluster of clusters.slice(0, 2)) {
+    hints.push(
+      `text movement cluster: ${formatQuotedList(cluster.texts)} dx=${formatRange(cluster.xRange)}px ` +
+        `dy=${formatRange(cluster.yRange)}px`,
+    );
+  }
+
+  const addedText = data.ocr?.addedText ?? [];
+  if (addedText.length > 0) {
+    hints.push(`added text candidates: ${formatTextChanges(addedText)}`);
+  }
+
+  const removedText = data.ocr?.removedText ?? [];
+  if (removedText.length > 0) {
+    hints.push(`removed text candidates: ${formatTextChanges(removedText)}`);
+  }
+
+  const controlDeltas = (data.nonTextDeltas ?? [])
+    .filter((delta) => ['icon', 'toggle', 'chevron', 'separator'].includes(delta.likelyKind))
+    .slice(0, 3);
+  if (controlDeltas.length > 0) {
+    hints.push(`non-text controls/boundaries: ${controlDeltas.map(formatNonTextHint).join('; ')}`);
+  }
+
+  const largestRegion = data.regions?.[0];
+  if (largestRegion) {
+    hints.push(
+      `largest changed region: r${largestRegion.index} ${largestRegion.location} ` +
+        `${largestRegion.shareOfDiffPercentage}% of diff, ${largestRegion.dominantChange}`,
+    );
+  }
+
+  return hints.slice(0, 6);
+}
+
+function formatTextChanges(
+  changes: Array<{ text: string; rect: { x: number; y: number; width: number; height: number } }>,
+): string {
+  return changes
+    .slice(0, 3)
+    .map((change) => `${JSON.stringify(change.text)} at x=${change.rect.x},y=${change.rect.y}`)
+    .join(', ');
+}
+
+function formatNonTextHint(delta: {
+  likelyKind: string;
+  nearestText?: string;
+  regionIndex?: number;
+}): string {
+  const anchor = delta.nearestText ? ` near ${JSON.stringify(delta.nearestText)}` : '';
+  const region = delta.regionIndex ? ` r${delta.regionIndex}` : '';
+  return `${delta.likelyKind}${anchor}${region}`;
+}
+
+function formatRange(range: { min: number; max: number }): string {
+  return range.min === range.max
+    ? formatSignedPixels(range.min)
+    : `${formatSignedPixels(range.min)}..${formatSignedPixels(range.max)}`;
+}
+
+function formatQuotedList(values: string[]): string {
+  const shown = values.slice(0, 4).map((value) => JSON.stringify(value));
+  const suffix = values.length > shown.length ? ` +${values.length - shown.length} more` : '';
+  return `${shown.join(', ')}${suffix}`;
+}
+
 function formatScreenshotRegionDetails(region: ScreenshotDiffRegion): string | null {
   const details = [
     region.size ? `size=${region.size}` : null,