feat: add skillgym tests

Author: thymikee

.gitignore

@@ -28,3 +28,4 @@ xcuserdata/
 *.app
 *.xctestrun
 *.xcarchive
+.skillgym-results/

README.md

@@ -87,6 +87,11 @@ For people:
 
 - [Website](https://agent-device.dev/)
 - [Docs](https://incubator.callstack.com/agent-device/docs/introduction)
+- [Skillgym starter](test/skillgym/README.md)
+
+Local benchmark starter:
+
+- `pnpm test:skillgym`
 
 For agents:
 

examples/test-app/.gitignore

@@ -0,0 +1,2 @@
+.expo/
+node_modules/

examples/test-app/README.md

@@ -0,0 +1,78 @@
+# Agent Device Tester
+
+`Agent Device Tester` is a minimal Expo Router fixture app for `agent-device` and `skillgym` experiments.
+
+It is intentionally small, but each surface is dense with durable accessibility targets so a few screens cover a large share of the workflows we care about.
+
+## Why this app exists
+
+- It gives `agent-device` a stable React Native target that we control.
+- It makes `skillgym` prompts concrete: the agent can inspect real app files instead of answering against an imagined UI.
+- It keeps the number of screens low while still covering roughly 50 practical interaction and verification cases.
+
+## Screens
+
+- `Home`: visible-text checks, dismissible banner, modal open/close, async loading, status badge, switch state
+- `Catalog`: search debounce, filter chips, long-list scroll, favorite toggles, cart updates, drill-in navigation
+- `Product detail`: back navigation, quantity stepper, multiline notes, save action
+- `Checkout form`: required-field validation, fill vs type, checkbox state, choice groups, keyboard dismiss, success summary
+- `Settings`: switch rows, accordion content, loading and error states, retry flow, destructive-confirm modal
+
+Navigation uses Expo Router native bottom tabs, so the tab bar itself is also part of the test surface.
+
+## Coverage map
+
+These are the main case families this app can support without adding more screens:
+
+- app open and close
+- visible text verification with plain `snapshot`
+- interactive discovery with `snapshot -i`
+- `press` on stable buttons, pills, and rows
+- `fill` on single-line and multiline fields
+- `type` after focus for append flows
+- `get text` on headings, badges, summaries, and accordion content
+- `is visible` and `is exists` assertions
+- `wait` for async loading and success states
+- `diff snapshot` after dismissals and submits
+- long-list scrolling and `scrollintoview`
+- selector-based navigation across repeated cards
+- modal open, cancel, and confirm flows
+- switch and checkbox state changes
+- validation-error and recovery loops
+- retryable error banners
+- cart counters and quantity changes
+- screenshot and recording proof capture
+
+## Run locally
+
+From the repo root:
+
+```bash
+pnpm test-app:install
+pnpm test-app:ios
+```
+
+Or on Android:
+
+```bash
+pnpm test-app:install
+pnpm test-app:android
+```
+
+If you prefer to work from inside the app folder:
+
+```bash
+cd examples/test-app
+pnpm install --ignore-workspace --lockfile=false
+pnpm ios
+```
+
+Or on Android:
+
+```bash
+cd examples/test-app
+pnpm install --ignore-workspace --lockfile=false
+pnpm android
+```
+
+Once the app is running, use `agent-device` against `Agent Device Tester` like any other target app.

examples/test-app/app.json

@@ -0,0 +1,20 @@
+{
+  "expo": {
+    "name": "Agent Device Tester",
+    "slug": "agent-device-test-app",
+    "version": "1.0.0",
+    "orientation": "portrait",
+    "userInterfaceStyle": "automatic",
+    "newArchEnabled": true,
+    "plugins": ["expo-router"],
+    "ios": {
+      "supportsTablet": true,
+      "bundleIdentifier": "com.callstack.agentdevicelab"
+    },
+    "android": {
+      "package": "com.callstack.agentdevicelab",
+      "edgeToEdgeEnabled": true,
+      "predictiveBackGestureEnabled": false
+    }
+  }
+}

examples/test-app/app/(tabs)/_layout.tsx

@@ -0,0 +1,43 @@
+import { NativeTabs } from 'expo-router/unstable-native-tabs';
+
+import { useLabState } from '../../src/lab-state';
+import { useAppColors } from '../../src/theme';
+
+export default function TabsLayout() {
+  const colors = useAppColors();
+  const { cartCount, diagnosticsState } = useLabState();
+
+  return (
+    <NativeTabs
+      backgroundColor={colors.tabBar}
+      badgeBackgroundColor={colors.accent}
+      iconColor={{ default: colors.textSoft, selected: colors.accent }}
+      labelStyle={{
+        default: { color: colors.textSoft, fontSize: 11, fontWeight: '600' },
+        selected: { color: colors.accent, fontSize: 11, fontWeight: '700' },
+      }}
+      tintColor={colors.accent}
+    >
+      <NativeTabs.Trigger name="index">
+        <NativeTabs.Trigger.Icon md="home" sf="house.fill" />
+        <NativeTabs.Trigger.Label>Home</NativeTabs.Trigger.Label>
+      </NativeTabs.Trigger>
+      <NativeTabs.Trigger name="catalog">
+        <NativeTabs.Trigger.Icon md="storefront" sf="square.grid.2x2.fill" />
+        <NativeTabs.Trigger.Label>Catalog</NativeTabs.Trigger.Label>
+        <NativeTabs.Trigger.Badge hidden={cartCount === 0}>
+          {String(cartCount)}
+        </NativeTabs.Trigger.Badge>
+      </NativeTabs.Trigger>
+      <NativeTabs.Trigger name="form">
+        <NativeTabs.Trigger.Icon md="fact_check" sf="doc.text.fill" />
+        <NativeTabs.Trigger.Label>Form</NativeTabs.Trigger.Label>
+      </NativeTabs.Trigger>
+      <NativeTabs.Trigger name="settings">
+        <NativeTabs.Trigger.Icon md="settings" sf="gearshape.fill" />
+        <NativeTabs.Trigger.Label>Settings</NativeTabs.Trigger.Label>
+        <NativeTabs.Trigger.Badge hidden={diagnosticsState !== 'error'}>!</NativeTabs.Trigger.Badge>
+      </NativeTabs.Trigger>
+    </NativeTabs>
+  );
+}

examples/test-app/app/(tabs)/catalog.tsx

@@ -0,0 +1,27 @@
+import { useRouter } from 'expo-router';
+
+import { AppFrame } from '../../src/components';
+import { useLabState } from '../../src/lab-state';
+import { CatalogScreen } from '../../src/screens/CatalogScreen';
+
+export default function CatalogRoute() {
+  const router = useRouter();
+  const state = useLabState();
+
+  return (
+    <AppFrame>
+      <CatalogScreen
+        activeCategory={state.activeCategory}
+        cart={state.cartCounts}
+        favorites={new Set(state.favoriteIds)}
+        onAddToCart={state.addToCart}
+        onOpenDetails={(productId) => router.push(`/product/${productId}`)}
+        onSearchDraftChange={state.setSearchDraft}
+        onSelectCategory={state.setActiveCategory}
+        onToggleFavorite={state.toggleFavorite}
+        products={state.catalogProducts}
+        searchDraft={state.searchDraft}
+      />
+    </AppFrame>
+  );
+}

examples/test-app/app/(tabs)/form.tsx

@@ -0,0 +1,20 @@
+import { AppFrame } from '../../src/components';
+import { useLabState } from '../../src/lab-state';
+import { FormScreen } from '../../src/screens/FormScreen';
+
+export default function FormRoute() {
+  const state = useLabState();
+
+  return (
+    <AppFrame>
+      <FormScreen
+        errors={state.formErrors}
+        form={state.form}
+        onChange={state.updateForm}
+        onReset={state.resetForm}
+        onSubmit={state.submitOrder}
+        submittedSummary={state.submittedSummary}
+      />
+    </AppFrame>
+  );
+}

examples/test-app/app/(tabs)/index.tsx

@@ -0,0 +1,31 @@
+import { useRouter } from 'expo-router';
+
+import { AppFrame } from '../../src/components';
+import { useLabState } from '../../src/lab-state';
+import { HomeScreen } from '../../src/screens/HomeScreen';
+
+export default function HomeRoute() {
+  const router = useRouter();
+  const state = useLabState();
+
+  return (
+    <AppFrame>
+      <HomeScreen
+        cartCount={state.cartCount}
+        isOnline={state.isOnline}
+        isRefreshing={state.isRefreshing}
+        lastSyncLabel={state.lastSyncLabel}
+        modalVisible={state.modalVisible}
+        noticeVisible={state.noticeVisible}
+        onDismissNotice={state.dismissNotice}
+        onHideModal={state.hideModal}
+        onOpenCatalog={() => router.navigate('/catalog')}
+        onOpenForm={() => router.navigate('/form')}
+        onOpenSettings={() => router.navigate('/settings')}
+        onRefresh={state.refreshMetrics}
+        onSetOnline={state.setIsOnline}
+        onShowModal={state.showModal}
+      />
+    </AppFrame>
+  );
+}

examples/test-app/app/(tabs)/settings.tsx

@@ -0,0 +1,28 @@
+import { AppFrame } from '../../src/components';
+import { useLabState } from '../../src/lab-state';
+import { SettingsScreen } from '../../src/screens/SettingsScreen';
+
+export default function SettingsRoute() {
+  const state = useLabState();
+
+  return (
+    <AppFrame>
+      <SettingsScreen
+        diagnosticsExpanded={state.diagnosticsExpanded}
+        diagnosticsLoading={state.diagnosticsLoading}
+        diagnosticsState={state.diagnosticsState}
+        notificationsEnabled={state.notificationsEnabled}
+        onConfirmReset={state.resetLabState}
+        onHideResetModal={state.hideResetModal}
+        onLoadDiagnostics={state.loadDiagnostics}
+        onRetryDiagnostics={state.retryDiagnostics}
+        onSetNotificationsEnabled={state.setNotificationsEnabled}
+        onSetReducedMotionEnabled={state.setReducedMotionEnabled}
+        onShowResetModal={state.showResetModal}
+        onToggleDiagnostics={() => state.setDiagnosticsExpanded(!state.diagnosticsExpanded)}
+        reducedMotionEnabled={state.reducedMotionEnabled}
+        resetModalVisible={state.resetModalVisible}
+      />
+    </AppFrame>
+  );
+}