Improve integration failure diagnostics (#40)

Author: thymikee

.github/workflows/android.yml

@@ -54,4 +54,5 @@ jobs:
           path: |
             ${{ steps.android-agent-home.outputs.dir }}/daemon.log
             ${{ steps.android-agent-home.outputs.dir }}/sessions/**
+            test/artifacts/**
             test/screenshots/**

.github/workflows/ios.yml

@@ -98,4 +98,5 @@ jobs:
           path: |
             ${{ steps.ios-agent-home.outputs.dir }}/daemon.log
             ${{ steps.ios-agent-home.outputs.dir }}/sessions/**
+            test/artifacts/**
             test/screenshots/**

test/integration/android.test.ts

@@ -1,6 +1,5 @@
 import test from 'node:test';
-import assert from 'node:assert/strict';
-import { formatResultDebug, runCliJson } from './test-helpers.ts';
+import { createIntegrationTestContext, runCliJson } from './test-helpers.ts';
 
 const session = ['--session', 'android-test'];
 
@@ -9,35 +8,34 @@ test.after(() => {
 });
 
 test('android settings commands', () => {
-  const openArgs = ['open', 'Settings', '--platform', 'android', ...session];
-  const open = runCliJson(openArgs);
-  assert.equal(open.status, 0, formatResultDebug('open settings', openArgs, open));
+  const integration = createIntegrationTestContext({
+    platform: 'android',
+    testName: 'android settings commands',
+  });
+  const openArgs = ['open', 'Settings', '--platform', 'android', '--json', ...session];
+  integration.runStep('open settings', openArgs);
 
   const snapshotArgs = ['snapshot', '-i', '--json', ...session];
-  const snapshot = runCliJson(snapshotArgs);
-  assert.equal(snapshot.status, 0, formatResultDebug('snapshot', snapshotArgs, snapshot));
-  assert.ok(
-    Array.isArray(snapshot.json?.data?.nodes),
-    formatResultDebug('snapshot nodes', snapshotArgs, snapshot),
-  );
+  const snapshot = integration.runStep('snapshot', snapshotArgs);
+  integration.assertResult(Array.isArray(snapshot.json?.data?.nodes), 'snapshot nodes', snapshotArgs, snapshot, {
+    detail: 'expected snapshot to include a nodes array',
+  });
 
-  const clickAppsArgs = ['click', '@e13', ...session];
-  const clickApps = runCliJson(clickAppsArgs);
-  assert.equal(clickApps.status, 0, formatResultDebug('click apps', clickAppsArgs, clickApps));
+  const clickAppsArgs = ['click', '@e13', '--json', ...session];
+  integration.runStep('click apps', clickAppsArgs);
 
   const snapshotAppsArgs = ['snapshot', '-i', '--json', ...session];
-  const snapshotApps = runCliJson(snapshotAppsArgs);
-  assert.equal(
-    snapshotApps.status,
-    0,
-    formatResultDebug('snapshot apps', snapshotAppsArgs, snapshotApps),
-  );
-  assert.ok(
+  const snapshotApps = integration.runStep('snapshot apps', snapshotAppsArgs);
+  integration.assertResult(
     Array.isArray(snapshotApps.json?.data?.nodes),
-    formatResultDebug('snapshot apps nodes', snapshotAppsArgs, snapshotApps),
+    'snapshot apps nodes',
+    snapshotAppsArgs,
+    snapshotApps,
+    {
+      detail: 'expected snapshot after click to include a nodes array',
+    },
   );
 
-  const backArgs = ['back', ...session];
-  const back = runCliJson(backArgs);
-  assert.equal(back.status, 0, formatResultDebug('back', backArgs, back));
+  const backArgs = ['back', '--json', ...session];
+  integration.runStep('back', backArgs);
 });

test/integration/ios.test.ts

@@ -1,8 +1,7 @@
 import test from 'node:test';
-import assert from 'node:assert/strict';
 import { existsSync } from 'node:fs';
 import path from 'node:path';
-import { formatResultDebug, runCliJson } from './test-helpers.ts';
+import { createIntegrationTestContext, runCliJson } from './test-helpers.ts';
 
 const session = ['--session', 'ios-test'];
 
@@ -11,53 +10,60 @@ test.after(() => {
 });
 
 test('ios settings commands', { skip: shouldSkipIos() }, async () => {
+  const integration = createIntegrationTestContext({
+    platform: 'ios',
+    testName: 'ios settings commands',
+  });
   const openArgs = ['open', 'com.apple.Preferences', '--platform', 'ios', '--json', ...session];
-  const open = runCliJson(openArgs);
-  assert.equal(open.status, 0, formatResultDebug('open settings', openArgs, open));
+  integration.runStep('open settings', openArgs);
 
   const outPath = path.resolve('test/screenshots/ios-settings.png');
   const shotArgs = ['screenshot', outPath, '--platform', 'ios', '--json', ...session];
-  const shot = runCliJson(shotArgs);
-  assert.equal(shot.status, 0, formatResultDebug('screenshot settings', shotArgs, shot));
-  assert.ok(existsSync(outPath), formatResultDebug('screenshot file missing', shotArgs, shot));
+  const shot = integration.runStep('screenshot settings', shotArgs);
+  integration.assertResult(existsSync(outPath), 'screenshot file missing', shotArgs, shot, {
+    detail: `expected screenshot file at ${outPath}`,
+  });
 
   const snapshotArgs = ['snapshot', '-i', '--json', ...session];
-  const snapshot = runCliJson(snapshotArgs);
-  assert.equal(snapshot.status, 0, formatResultDebug('snapshot', snapshotArgs, snapshot));
-  assert.ok(
-    Array.isArray(snapshot.json?.data?.nodes),
-    formatResultDebug('snapshot nodes', snapshotArgs, snapshot),
-  );
+  const snapshot = integration.runStep('snapshot', snapshotArgs);
+  integration.assertResult(Array.isArray(snapshot.json?.data?.nodes), 'snapshot nodes', snapshotArgs, snapshot, {
+    detail: 'expected snapshot to include a nodes array',
+  });
 
   const clickArgs = ['click', '@e21', '--json', ...session];
-  const click = runCliJson(clickArgs);
-  assert.equal(click.status, 0, formatResultDebug('click @e13', clickArgs, click));
+  integration.runStep('click @e21', clickArgs);
 
-  const snapshotGeneral = runCliJson(['snapshot', '--json', ...session]);
+  const snapshotGeneralArgs = ['snapshot', '--json', ...session];
+  const snapshotGeneral = integration.runStep('snapshot general', snapshotGeneralArgs);
   const generalDescription = 'Manage your overall setup and preferences';
   const generalNodes = Array.isArray(snapshotGeneral.json?.data?.nodes)
     ? snapshotGeneral.json.data.nodes
     : [];
-  assert.ok(
+  integration.assertResult(
     generalNodes.some(
       (node: { label?: string }) =>
         typeof node?.label === 'string' && node.label.includes(generalDescription),
     ),
-    formatResultDebug('snapshot shows general page description', snapshotArgs, snapshotGeneral),
+    'snapshot shows general page description',
+    snapshotGeneralArgs,
+    snapshotGeneral,
+    {
+      detail: `expected a node label containing ${JSON.stringify(generalDescription)}`,
+    },
   );
 
   const findTextArgs = ['find', 'text', generalDescription, 'exists', '--json', ...session];
-  const findText = runCliJson(findTextArgs);
-  assert.equal(findText.status, 0, formatResultDebug('find text', findTextArgs, findText));
-  assert.equal(
+  const findText = integration.runStep('find text', findTextArgs);
+  integration.assertResult(
     findText.json?.success,
-    true,
-    formatResultDebug('find text success', findTextArgs, findText),
+    'find text success',
+    findTextArgs,
+    findText,
+    { detail: 'expected find command to return success=true' },
   );
 
-  const backArgs = ['back', ...session];
-  const back = runCliJson(backArgs);
-  assert.equal(back.status, 0, formatResultDebug('back', backArgs, back));
+  const backArgs = ['back', '--json', ...session];
+  integration.runStep('back', backArgs);
 });
 
 function shouldSkipIos(): boolean {