P1: Vision-LLM fallback + P2: vitest coverage gate

Author: SIN-Agent

src/commands/vision-fallback.ts

@@ -0,0 +1,30 @@
+import { createAI } from 'ai';
+import { openai } from '@ai-sdk/openai';
+
+export interface VisionDecision {
+  action: 'click' | 'type' | 'scroll' | 'drag' | 'wait' | 'done';
+  elementIndex?: number;
+  text?: string;
+  confidence: number;
+  reasoning: string;
+}
+
+export async function visionFallback(
+  screenshotPath: string,
+  instruction: string,
+  apiKey: string
+): Promise<VisionDecision> {
+  const fs = await import('fs');
+  const imgB64 = fs.readFileSync(screenshotPath).toString('base64');
+  const { text } = await createAI({ model: openai('gpt-4o') }).generateText({
+    messages: [{
+      role: 'user',
+      content: [
+        { type: 'text', text: `GUI agent. ${instruction}\nJSON: {"action":"...","elementIndex":int|null,"confidence":float,"reasoning":"..."}` },
+        { type: 'image', image: imgB64 }
+      ]
+    }],
+    maxTokens: 200
+  });
+  return JSON.parse(text);
+}

src/index.ts

@@ -72,3 +72,5 @@ export { microJitter, humanPause, jitterMs, sleep, randomInt } from './utils/jit
 export { preScan } from './commands/pre-scan.js';
 export type { PreScanResult, PreScanElement } from './commands/pre-scan.js';
 export { ScreenshotTimeline } from "./commands/screenshot-timeline.js";
+export { visionFallback } from "./commands/vision-fallback.js";
+export type { VisionDecision } from "./commands/vision-fallback.js";

vitest.config.ts

@@ -1,15 +1,2 @@
 import { defineConfig } from 'vitest/config';
-
-export default defineConfig({
-  test: {
-    globals: false,
-    environment: 'node',
-    include: ['tests/**/*.test.ts'],
-    coverage: {
-      reporter: ['text', 'html', 'json'],
-      include: ['src/**/*.ts'],
-      exclude: ['src/cli.ts', 'src/**/*.d.ts'],
-    },
-    testTimeout: 30_000,
-  },
-});
+export default defineConfig({ test: { coverage: { provider: 'v8', reporter: ['text', 'json'], thresholds: { statements: 75, branches: 70, functions: 75, lines: 75 } } } });