refactor(core): update describeElementAtPoint integration;move recorder describer out of core (#2692)

Move recorder UI describer implementation and tests into playground.

- Keep describeElementAtPoint and locatorVerifyFn exposed from core as
- general-purpose element description utilities.

- Update Studio recorder codegen to import recorder describer from playground.

Author: fishear

apps/studio/rsbuild.config.ts

@@ -153,6 +153,7 @@ export default defineConfig({
           '@midscene/harmony',
           '@midscene/ios',
           '@midscene/playground',
+          '@midscene/playground/recorder-ui-describer',
         ],
         sourceMap: true,
       },

apps/studio/src/main/recorder/codegen.ts

@@ -1,10 +1,10 @@
 import {
   convertRecordLogIntoMarkdown,
-  describeRecorderUIEvents,
   generatePlaywrightTest,
   generateRecorderSessionMetadata,
   generateRecorderYamlTest,
 } from '@midscene/core/ai-model';
+import { describeRecorderUIEvents } from '@midscene/playground/recorder-ui-describer';
 import { getDebug } from '@midscene/shared/logger';
 import type {
   DescribeRecorderUIEventsRequest,

apps/studio/src/renderer/recorder/StudioRecorderProvider.tsx

@@ -980,6 +980,28 @@ export function StudioRecorderProvider({ children }: PropsWithChildren) {
                   result.trace.annotatedScreenshotRef?.path,
               }
             : undefined;
+        const elementDescription = result.trace?.elementDescription?.trim();
+        if (elementDescription && result.trace?.verifyPassed === false) {
+          const semanticAction = buildRecorderSemanticAction(event);
+          return normalizeInputRecorderSemantic({
+            ...event,
+            semantic: {
+              source: 'aiDescribe',
+              status: 'ready',
+              elementDescription,
+              replayInstruction: buildMidsceneRecorderReplayInstruction(
+                semanticAction,
+                elementDescription,
+              ),
+              actionSummary: buildMidsceneRecorderActionSummary(
+                semanticAction,
+                elementDescription,
+              ),
+              confidence: 'low',
+              ...(aiDescribe ? { aiDescribe } : {}),
+            },
+          });
+        }
         return {
           ...event,
           semantic: {

apps/studio/tests/studio-recorder-main-codegen.test.ts

@@ -16,6 +16,10 @@ vi.mock('@midscene/core/ai-model', () => ({
   ),
 }));
 
+vi.mock('@midscene/playground/recorder-ui-describer', () => ({
+  describeRecorderUIEvents: vi.fn(async () => []),
+}));
+
 import {
   convertRecordLogIntoMarkdown,
   generatePlaywrightTest,

apps/studio/tests/studio-recorder-provider.test.tsx

@@ -382,15 +382,15 @@ describe('StudioRecorderProvider preview recording', () => {
     }
   });
 
-  it('falls back from aiDescribe failed events to recorderAI descriptions', async () => {
+  it('falls back from truly failed aiDescribe events to recorderAI descriptions', async () => {
     const event = {
       type: 'click',
       source: 'studio-preview',
       actionType: 'Click',
       semantic: {
         source: 'aiDescribe',
         status: 'failed',
-        error: 'aiDescribe verification failed.',
+        error: 'Active agent does not support describeElementAtPoint.',
       },
       elementRect: { x: 10, y: 20 },
       pageInfo: { width: 1200, height: 800 },
@@ -420,7 +420,69 @@ describe('StudioRecorderProvider preview recording', () => {
         fallbackFrom: {
           source: 'aiDescribe',
           status: 'failed',
-          error: 'aiDescribe verification failed.',
+          error: 'Active agent does not support describeElementAtPoint.',
+        },
+      },
+    });
+
+    await mounted.cleanup();
+  });
+
+  it('keeps aiDescribe when only verification fails and does not fallback to recorderAI', async () => {
+    const event = {
+      type: 'click',
+      source: 'studio-preview',
+      actionType: 'Click',
+      elementRect: { x: 10, y: 20 },
+      pageInfo: { width: 1200, height: 800 },
+      screenshotAfter: 'data:image/png;base64,shot',
+      timestamp: 123,
+      hashId: 'click-ai-describe-verify-failed',
+    };
+    const describeRecorderEventAtPoint = vi.fn(async () => ({
+      ok: false,
+      error: 'aiDescribe verification failed.',
+      trace: {
+        elementDescription: 'Submit button in the form footer',
+        verifyPassed: false,
+        centerDistance: 24,
+        point: [10, 20],
+        verifyResult: {
+          pass: false,
+          center: [30, 20],
+          centerDistance: 24,
+        },
+        annotatedScreenshotRef: {
+          path: '/tmp/verify-failed-annotated.png',
+        },
+      },
+    }));
+    const { context } = createConnectedStudioContext({
+      events: [event],
+      describeRecorderEventAtPoint,
+    });
+    const mounted = await mountRecorder(context);
+
+    await act(async () => {
+      await mounted.recorder?.startRecording();
+    });
+    await flushPromises();
+    await flushPromises();
+
+    expect(describeStudioRecorderEventsWithAI).not.toHaveBeenCalled();
+    expect(mounted.recorder?.currentSession?.events[0]).toMatchObject({
+      hashId: 'click-ai-describe-verify-failed',
+      semantic: {
+        source: 'aiDescribe',
+        status: 'ready',
+        elementDescription: 'Submit button in the form footer',
+        aiDescribe: {
+          verifyPrompt: true,
+          verifyPassed: false,
+          centerDistance: 24,
+          expectedCenter: [10, 20],
+          actualCenter: [30, 20],
+          annotatedScreenshotPath: '/tmp/verify-failed-annotated.png',
         },
       },
     });
@@ -1140,7 +1202,7 @@ describe('StudioRecorderProvider preview recording', () => {
         semantic: {
           source: 'aiDescribe',
           status: 'failed',
-          error: 'aiDescribe verification failed.',
+          error: 'Active agent does not support describeElementAtPoint.',
         },
       },
     }));
@@ -1238,7 +1300,7 @@ describe('StudioRecorderProvider preview recording', () => {
         fallbackFrom: {
           source: 'aiDescribe',
           status: 'failed',
-          error: 'aiDescribe verification failed.',
+          error: 'Active agent does not support describeElementAtPoint.',
         },
       },
     });

apps/studio/vitest.config.ts

@@ -9,6 +9,10 @@ export default defineConfig({
       '@preload': path.resolve(__dirname, 'src/preload'),
       '@renderer': path.resolve(__dirname, 'src/renderer'),
       '@shared': path.resolve(__dirname, 'src/shared'),
+      '@midscene/playground/recorder-ui-describer$': path.resolve(
+        __dirname,
+        '../../packages/playground/src/recorder-ui-describer.ts',
+      ),
     },
   },
   test: {