Merge pull request #38 from Lykhoyda/enhance/b120-screenshot-resize

feat(b120): downscale device_screenshot via macOS sips — closes #36

Author: Lykhoyda

.claude-plugin/marketplace.json

@@ -9,7 +9,7 @@
     {
       "name": "rn-dev-agent",
       "description": "AI agent that fully tests React Native features on simulator/emulator — navigates the app, verifies UI, walks user flows, and confirms internal state.",
-      "version": "0.25.1",
+      "version": "0.26.0",
       "source": "./",
       "category": "mobile-development",
       "homepage": "https://github.com/Lykhoyda/rn-dev-agent"

.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "rn-dev-agent",
-  "version": "0.25.1",
+  "version": "0.26.0",
   "description": "AI agent that fully tests React Native features on simulator/emulator — navigates the app, verifies UI, walks user flows, and confirms internal state.",
   "author": {
     "name": "Anton Lykhoyda",

scripts/cdp-bridge/dist/index.js

@@ -285,10 +285,12 @@ trackedTool('collect_logs', 'Collect logs from multiple sources in parallel: JS
 }, createCollectLogsHandler(getClient));
 // --- agent-device tools (native device interaction) ---
 trackedTool('device_list', 'List all available iOS simulators and Android emulators. Returns device name, UDID, platform, and status. Use before device_snapshot action=open to confirm the target device.', {}, createDeviceListHandler());
-trackedTool('device_screenshot', 'Capture a screenshot of the active device screen. Returns image data or file path. Prefer JPEG for faster capture. When both iOS sim and Android emulator are booted, defaults to the platform of the currently connected CDP target; override with `platform` if needed.', {
+trackedTool('device_screenshot', 'Capture a screenshot of the active device screen. Returns the file path. Prefer JPEG for faster capture. When both iOS sim and Android emulator are booted, defaults to the platform of the currently connected CDP target. Output is auto-downscaled to maxWidth (default 800px) via macOS sips to keep LLM context costs predictable; pass maxWidth=0 to disable when full-resolution capture is needed (visual diffing). meta.resize describes what happened.', {
     path: z.string().optional().describe('Output file path (default: auto-generated in /tmp). Use .jpg extension for JPEG.'),
     format: z.enum(['jpeg', 'png']).optional().describe('Image format (default: auto-detect from path extension, or jpeg)'),
     platform: z.enum(['ios', 'android']).optional().describe('Target device platform. Defaults to the currently-connected CDP target platform.'),
+    maxWidth: z.number().int().min(0).optional().describe('Downscale image so width does not exceed this many pixels. 0 disables resize. Default 800 (saves ~46% on iPhone 15/17 Pro screenshots without losing label readability).'),
+    quality: z.number().int().min(1).max(100).optional().describe('JPEG compression quality (1-100). Only applied to .jpg/.jpeg files. Default 85.'),
 }, createDeviceScreenshotHandler(getClient));
 trackedTool('device_snapshot', 'Manage device sessions and capture UI snapshots. action=open starts a session (required before other device_ tools). action=snapshot returns the accessibility tree with @ref identifiers for device_press/device_fill. action=close ends the session. Use attachOnly=true on action=open to skip launching the app when it is already running (avoids relaunch-induced bundle races — B112).', {
     action: z.enum(['open', 'close', 'snapshot']).default('snapshot').describe('open: start session for an app. snapshot: capture UI tree with element refs. close: end session.'),

scripts/cdp-bridge/dist/tools/device-list.js

@@ -1,7 +1,19 @@
 import { runAgentDevice } from '../agent-device-wrapper.js';
+import { resizeWithSips } from './device-screenshot-resize.js';
 export function createDeviceListHandler() {
     return async () => runAgentDevice(['devices'], { skipSession: true });
 }
+/**
+ * Pure derivation of the output path for a screenshot call. Extracted so the
+ * handler can know the path independently from `buildScreenshotArgs` (used to
+ * pass it to the post-resize step) and to keep `buildScreenshotArgs` tests stable.
+ */
+export function deriveScreenshotPath(args, now = Date.now) {
+    if (args.path)
+        return args.path;
+    const ext = args.format === 'jpeg' ? 'jpg' : args.format === 'png' ? 'png' : 'jpg';
+    return `/tmp/rn-screenshot-${now()}.${ext}`;
+}
 /**
  * B113 fix (D636): agent-device >= 0.8.0 exposes only `[path]` and `--out <path>`
  * — no `--format`. Emitting --format caused 100% failure ("Unknown flag: --format").
@@ -11,22 +23,88 @@ export function createDeviceListHandler() {
  * Exported for unit tests — pure function, no I/O.
  */
 export function buildScreenshotArgs(args, now = Date.now) {
-    let outputPath = args.path;
-    if (!outputPath) {
-        const ext = args.format === 'jpeg' ? 'jpg' : args.format === 'png' ? 'png' : 'jpg';
-        outputPath = `/tmp/rn-screenshot-${now()}.${ext}`;
+    return ['screenshot', '--out', deriveScreenshotPath(args, now)];
+}
+/**
+ * B120 / GH #36: extract the path agent-device actually wrote to. Daemon and
+ * Swift-runner paths echo it via `data.path`; fast-runner uses its own tmp
+ * file (`/tmp/rn-fast-screenshot-*.png`) regardless of `--out`, also exposed
+ * via `data.path`. Falls back to the requested path when the response shape
+ * is unexpected so resize still has a target to attempt.
+ */
+export function resolveScreenshotPath(result, fallback) {
+    try {
+        const envelope = JSON.parse(result.content[0].text);
+        const candidate = envelope?.data?.path;
+        if (typeof candidate === 'string' && candidate.startsWith('/')) {
+            return candidate;
+        }
+    }
+    catch { /* malformed envelope — use fallback */ }
+    return fallback;
+}
+export function wrapResultWithResize(result, resize) {
+    if (result.isError)
+        return result;
+    try {
+        const envelope = JSON.parse(result.content[0].text);
+        const resizeMeta = { resized: resize.resized };
+        if (resize.resized) {
+            if (resize.originalDims)
+                resizeMeta.fromDims = resize.originalDims;
+            if (resize.newDims)
+                resizeMeta.toDims = resize.newDims;
+            if (resize.originalBytes !== undefined)
+                resizeMeta.fromBytes = resize.originalBytes;
+            if (resize.newBytes !== undefined)
+                resizeMeta.toBytes = resize.newBytes;
+            if (resize.originalBytes && resize.newBytes) {
+                resizeMeta.savedPercent = Math.round((1 - resize.newBytes / resize.originalBytes) * 100);
+            }
+        }
+        else if (resize.reason) {
+            resizeMeta.reason = resize.reason;
+        }
+        envelope.meta = { ...envelope.meta, resize: resizeMeta };
+        if (envelope.data && resize.resized) {
+            envelope.data.path = resize.path;
+        }
+        return { content: [{ type: 'text', text: JSON.stringify(envelope) }] };
+    }
+    catch {
+        return result;
     }
-    return ['screenshot', '--out', outputPath];
 }
 /**
- * B117/D638: device_screenshot now accepts an optional `platform` and, when not
+ * B117/D638: device_screenshot accepts an optional `platform` and, when not
  * provided, falls back to the current CDP target's platform. Prevents
  * wrong-device screenshots when both iOS sim and Android emulator are booted.
+ *
+ * B120 / GH #36: post-process via macOS `sips` to downscale native-resolution
+ * images that otherwise blow LLM context budgets. Defaults to maxWidth=1200,
+ * quality=85 (JPEG only). Set maxWidth=0 to disable. Gracefully degrades on
+ * non-macOS / missing sips — original screenshot is still returned with a
+ * `meta.resize.reason` explaining why.
+ *
  * getClient is optional so existing callers/tests still compile.
  */
 export function createDeviceScreenshotHandler(getClient) {
     return async (args) => {
         const platform = args.platform ?? getClient?.()?.connectedTarget?.platform ?? null;
-        return runAgentDevice(buildScreenshotArgs(args), { platform });
+        const requestedPath = deriveScreenshotPath(args);
+        // Pin the path explicitly so the post-resize step targets the same file
+        // regardless of which dispatch tier (fast-runner / daemon / CLI) responded.
+        const argsWithPath = { ...args, path: requestedPath };
+        const result = await runAgentDevice(buildScreenshotArgs(argsWithPath), { platform });
+        if (result.isError)
+            return result;
+        const actualPath = resolveScreenshotPath(result, requestedPath);
+        const resizeOpts = {};
+        if (args.maxWidth !== undefined)
+            resizeOpts.maxWidth = args.maxWidth;
+        if (args.quality !== undefined)
+            resizeOpts.quality = args.quality;
+        const resize = await resizeWithSips(actualPath, resizeOpts);
+        return wrapResultWithResize(result, resize);
     };
 }

scripts/cdp-bridge/dist/tools/device-screenshot-resize.js

@@ -0,0 +1,119 @@
+import { execFile as execFileCb } from 'node:child_process';
+import { promisify } from 'node:util';
+import { statSync } from 'node:fs';
+const execFile = promisify(execFileCb);
+/**
+ * B120 / GH #36: defaults validated against a live iPhone 17 Pro screenshot
+ * (native 1206×2622, ~193 KB JPEG). Empirical measurements:
+ *   maxWidth=1200 → 181 KB (−7%, near no-op on modern iPhones)
+ *   maxWidth=1000 → 144 KB (−25%)
+ *   maxWidth=800  → 105 KB (−46%)  ← matches the issue's suggested default
+ *   maxWidth=600  →  68 KB (−65%)
+ * Picked 800 to match the issue's suggestion and produce meaningful savings
+ * even on devices whose native width is already close to 1200. Still leaves
+ * text labels at ~35-40 px tall — comfortably readable for any agent task.
+ * Set maxWidth=0 to disable when full-resolution capture is required (visual
+ * diffing, pixel comparisons).
+ */
+export const DEFAULT_MAX_WIDTH = 800;
+export const DEFAULT_QUALITY = 85;
+let sipsAvailable = null;
+const defaultFileSize = (path) => {
+    try {
+        return statSync(path).size;
+    }
+    catch {
+        return undefined;
+    }
+};
+async function checkSipsAvailable(deps) {
+    if (sipsAvailable !== null)
+        return sipsAvailable;
+    const runner = deps.exec ?? execFile;
+    try {
+        await runner('sips', ['--version'], { timeout: 1500 });
+        sipsAvailable = true;
+    }
+    catch {
+        sipsAvailable = false;
+    }
+    return sipsAvailable;
+}
+/** Test-only: reset the cached `sips --version` probe. */
+export function resetSipsProbeForTesting() {
+    sipsAvailable = null;
+}
+export function parseSipsDimensions(stdout) {
+    // sips -g pixelWidth -g pixelHeight <path> output:
+    //   /path/to/file.jpg
+    //     pixelWidth: 1179
+    //     pixelHeight: 2556
+    const wMatch = stdout.match(/pixelWidth:\s*(\d+)/);
+    const hMatch = stdout.match(/pixelHeight:\s*(\d+)/);
+    if (!wMatch || !hMatch)
+        return null;
+    return { width: parseInt(wMatch[1], 10), height: parseInt(hMatch[1], 10) };
+}
+async function getDimensions(path, deps) {
+    const runner = deps.exec ?? execFile;
+    try {
+        const { stdout } = await runner('sips', ['-g', 'pixelWidth', '-g', 'pixelHeight', path], { timeout: 5000, encoding: 'utf8' });
+        return parseSipsDimensions(stdout);
+    }
+    catch {
+        return null;
+    }
+}
+export function buildSipsResizeArgs(path, maxWidth, quality) {
+    const args = ['--resampleWidth', String(maxWidth)];
+    // JPEG quality only applies to .jpg/.jpeg files. sips accepts the flag for
+    // PNG paths but it's a no-op there — still safe, just noise — so we gate it
+    // on extension to keep the args minimal.
+    if (quality !== undefined && /\.jpe?g$/i.test(path)) {
+        args.push('-s', 'formatOptions', String(quality));
+    }
+    args.push(path);
+    return args;
+}
+/**
+ * Downscale an image at `path` in place via macOS `sips`. Returns details about
+ * what happened (or why no resize was performed). Never throws — degraded
+ * environments (Linux, missing sips, unreadable file) return `resized: false`
+ * with a reason, so the screenshot is still usable.
+ */
+export async function resizeWithSips(path, opts = {}, deps = {}) {
+    const maxWidth = opts.maxWidth ?? DEFAULT_MAX_WIDTH;
+    if (maxWidth <= 0) {
+        return { resized: false, path, reason: 'maxWidth-zero' };
+    }
+    if (!(await checkSipsAvailable(deps))) {
+        return { resized: false, path, reason: 'sips-unavailable' };
+    }
+    const originalDims = await getDimensions(path, deps);
+    if (!originalDims) {
+        return { resized: false, path, reason: 'no-dimensions' };
+    }
+    if (originalDims.width <= maxWidth) {
+        return { resized: false, path, reason: 'already-smaller', originalDims };
+    }
+    const fileSize = deps.fileSize ?? defaultFileSize;
+    const originalBytes = fileSize(path);
+    const runner = deps.exec ?? execFile;
+    const quality = opts.quality ?? DEFAULT_QUALITY;
+    try {
+        await runner('sips', buildSipsResizeArgs(path, maxWidth, quality), { timeout: 10_000 });
+    }
+    catch {
+        return { resized: false, path, reason: 'sips-failed', originalDims, originalBytes };
+    }
+    const newDims = await getDimensions(path, deps);
+    const newBytes = fileSize(path);
+    return {
+        resized: true,
+        path,
+        originalDims,
+        newDims: newDims ?? undefined,
+        originalBytes,
+        newBytes,
+    };
+}

scripts/cdp-bridge/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rn-dev-agent-cdp",
-  "version": "0.20.1",
+  "version": "0.21.0",
   "type": "module",
   "main": "dist/index.js",
   "scripts": {

scripts/cdp-bridge/src/index.ts

@@ -453,11 +453,13 @@ trackedTool(
 
 trackedTool(
   'device_screenshot',
-  'Capture a screenshot of the active device screen. Returns image data or file path. Prefer JPEG for faster capture. When both iOS sim and Android emulator are booted, defaults to the platform of the currently connected CDP target; override with `platform` if needed.',
+  'Capture a screenshot of the active device screen. Returns the file path. Prefer JPEG for faster capture. When both iOS sim and Android emulator are booted, defaults to the platform of the currently connected CDP target. Output is auto-downscaled to maxWidth (default 800px) via macOS sips to keep LLM context costs predictable; pass maxWidth=0 to disable when full-resolution capture is needed (visual diffing). meta.resize describes what happened.',
   {
     path: z.string().optional().describe('Output file path (default: auto-generated in /tmp). Use .jpg extension for JPEG.'),
     format: z.enum(['jpeg', 'png']).optional().describe('Image format (default: auto-detect from path extension, or jpeg)'),
     platform: z.enum(['ios', 'android']).optional().describe('Target device platform. Defaults to the currently-connected CDP target platform.'),
+    maxWidth: z.number().int().min(0).optional().describe('Downscale image so width does not exceed this many pixels. 0 disables resize. Default 800 (saves ~46% on iPhone 15/17 Pro screenshots without losing label readability).'),
+    quality: z.number().int().min(1).max(100).optional().describe('JPEG compression quality (1-100). Only applied to .jpg/.jpeg files. Default 85.'),
   },
   createDeviceScreenshotHandler(getClient),
 );