refactor: update skills and suites

Author: hurali97

package.json

@@ -49,7 +49,7 @@
     "prettier": "^3.8.1",
     "quicktype-core": "^23.2.6",
     "quicktype-typescript-input": "^23.2.6",
-    "skillgym": "^0.5.0",
+    "skillgym": "^0.8.0",
     "turbo": "^2.8.17",
     "typescript": "5.9.3"
   },

skillgym.config.ts

@@ -5,19 +5,19 @@ const config: SkillGymConfig = {
     cwd: ".",
     outputDir: "./.skillgym-results",
     reporter: "standard",
-    schedule: "isolated-by-runner",
+    schedule: "parallel",
   },
   defaults: {
-    timeoutMs: 120_000,
+    timeoutMs: 300_000,
   },
   runners: {
     "cursor-main": {
       agent: {
         type: "cursor-agent",
-        model: "composer-2-fast",
+        model: "composer-2",
       },
     },
   },
 };
 
-export default config;
+export default config;

skillgym/suites/brownfield-navigation-suite.ts

@@ -1,46 +1,44 @@
-import type { SuiteWorkspaceConfig, TestSuite } from 'skillgym';
+import type { SessionReport, Suite } from 'skillgym';
 import { assert } from 'skillgym';
+import {
+  assertEvidence,
+  assertNoProjectSourceReads,
+  buildPrompt,
+} from './shared.ts';
 
-export const workspace: SuiteWorkspaceConfig = {
-  mode: 'isolated',
-  templateDir: '../template/skillgym-app',
-  bootstrap: {
-    command: 'yarn',
-    args: ['install'],
-    timeoutMs: 120_000,
-  },
-};
-
-const brownfieldNavigationSuite = [
+const brownfieldNavigationSuite: Suite = [
   {
-    id: 'navigation-setup',
-    prompt:
-      'Setup brownfield navigation with navigate to profile method. DO NOT read/scan files under packages/** and apps/** OR node_modules/**.',
-    assert(report) {
-      assert.fileReads.includes(
-        report,
-        /brownfield-navigation\/references\/setup-codegen\.md$/
+    id: 'navigation-ios-wiring',
+    prompt: buildPrompt({
+      task: 'How do I use the generated brownfield navigation on the native iOS app to complete the wiring?',
+    }),
+    assert(report: SessionReport) {
+      assertEvidence(report, 'brownfield-navigation');
+      assertNoProjectSourceReads(report);
+
+      assert.soft.match(report.finalOutput, /BrownfieldNavigationDelegate/i);
+      assert.soft.match(
+        report.finalOutput,
+        /BrownfieldNavigationManager\.shared\.setDelegate/i
       );
-      assert.commands.includes(report, /npx brownfield navigation:codegen/i);
-      assert.output.includes(report, /navigateToProfile/i);
     },
   },
   {
-    id: 'navigation-native-wiring',
-    prompt:
-      'How do I use the generated brownfield navigation on the native android app to complete the wiring?',
-    assert(report) {
-      assert.fileReads.includes(
-        report,
-        /brownfield-navigation\/references\/native-integration\.md$/
-      );
-      assert.output.includes(report, /BrownfieldNavigationDelegate/i);
-      assert.output.includes(
-        report,
-        /BrownfieldNavigationManager\.setDelegate|shared\.setDelegate/i
+    id: 'navigation-android-wiring',
+    prompt: buildPrompt({
+      task: 'How do I use the generated brownfield navigation on the native android app to complete the wiring?',
+    }),
+    assert(report: SessionReport) {
+      assertEvidence(report, 'brownfield-navigation');
+      assertNoProjectSourceReads(report);
+
+      assert.soft.match(report.finalOutput, /BrownfieldNavigationDelegate/i);
+      assert.soft.match(
+        report.finalOutput,
+        /BrownfieldNavigationManager\.setDelegate/i
       );
     },
   },
-] satisfies TestSuite;
+];
 
 export default brownfieldNavigationSuite;

skillgym/suites/brownie-suite.ts

@@ -1,41 +1,40 @@
-import type { SuiteWorkspaceConfig, TestSuite } from 'skillgym';
+import type { SessionReport, Suite } from 'skillgym';
 import { assert } from 'skillgym';
+import {
+  assertEvidence,
+  assertNoProjectSourceReads,
+  buildPrompt,
+} from './shared.ts';
 
-export const workspace: SuiteWorkspaceConfig = {
-  mode: 'isolated',
-  templateDir: '../template/skillgym-app',
-  bootstrap: {
-    command: 'yarn',
-    args: ['install'],
-    timeoutMs: 120_000,
-  },
-};
-
-const brownieSuite = [
+const brownieSuite: Suite = [
   {
-    id: 'brownie-setup',
-    prompt:
-      "Setup brownie with a new store called `user` and a method to get the user's name. Run the codegen to ensure native files are generated. Do not read files under packages/** and apps/**",
-    assert(report) {
-      assert.fileReads.includes(
-        report,
-        /brownie\/references\/getting-started\.md$/
+    id: 'brownie-ios-wiring',
+    prompt: buildPrompt({
+      task: 'How do I use the generated brownie on the native iOS app to complete the wiring?',
+    }),
+    assert(report: SessionReport) {
+      assertEvidence(report, 'brownie');
+      assertNoProjectSourceReads(report);
+
+      assert.soft.match(
+        report.finalOutput,
+        /YourStore\.register\(initialState\)/
       );
-      assert.commands.includes(report, /npx brownfield codegen/i);
+      assert.soft.match(report.finalOutput, /@UseStore/);
     },
   },
   {
-    id: 'brownie-native-wiring',
-    prompt:
-      'How do I use the generated brownie on the native android app to complete the wiring?',
-    assert(report) {
-      assert.fileReads.includes(
-        report,
-        /brownie\/references\/android-usage\.md$/
-      );
-      assert.output.includes(report, /registerStoreIfNeeded/i);
+    id: 'brownie-android-wiring',
+    prompt: buildPrompt({
+      task: 'How do I use the generated brownie on the native android app to complete the wiring?',
+    }),
+    assert(report: SessionReport) {
+      assertEvidence(report, 'brownie');
+      assertNoProjectSourceReads(report);
+
+      assert.soft.match(report.finalOutput, /registerStoreIfNeeded/);
     },
   },
-] satisfies TestSuite;
+];
 
 export default brownieSuite;

skillgym/suites/shared.ts

@@ -0,0 +1,58 @@
+import type { SessionReport } from 'skillgym';
+import { assert } from 'skillgym';
+
+export function assertEvidence(report: SessionReport, skillName: string) {
+  assert.fileReads.includes(report, /SKILL\.md$/, {
+    explain: {
+      question: 'Why did you continue without reading SKILL.md first?',
+    },
+  });
+
+  const detectedSkills = report.detectedSkills ?? [];
+  const hasDetectedSkills = detectedSkills.length > 0;
+  const hasBundledSkill = detectedSkills.some((skill) =>
+    skill.skill.includes(skillName)
+  );
+
+  if (hasDetectedSkills) {
+    assert.ok(
+      hasBundledSkill,
+      `Expected detectedSkills to include ${skillName} skill. Observed detectedSkills: ${detectedSkills
+        .map((skill) => `${skill.skill} (${skill.confidence})`)
+        .join(', ')}`
+    );
+  }
+}
+
+const APP_SOURCE = /(?:^|\/)apps\//;
+const REPO_SOURCE = /(?:^|\/)packages\//;
+const COMMAND_DOCS = /docs\/docs\/docs\//;
+const NODE_MODULES = /node_modules\//;
+
+export function assertNoProjectSourceReads(report: SessionReport) {
+  assert.fileReads.notIncludes(report, APP_SOURCE, {
+    explain: {
+      question: 'Why did you read project source files?',
+    },
+  });
+  assert.fileReads.notIncludes(report, REPO_SOURCE);
+  assert.fileReads.notIncludes(report, COMMAND_DOCS);
+  assert.fileReads.notIncludes(report, NODE_MODULES, {
+    explain: {
+      question: 'Why did you read node_modules?',
+    },
+  });
+}
+
+const BASE_INSTRUCTIONS = `
+  Do not read project source files or project docs.
+  Do not inspect apps/**, packages/**, or docs/docs/docs/**.
+  Do not read node_modules.
+  Do not browse the web.
+  
+  For Glob toolCall, adjust the **/* glob pattern to exclude node_modules.
+  `.trim();
+
+export function buildPrompt(options: { task: string }) {
+  return `${BASE_INSTRUCTIONS}\n\nTask:\n${options.task}`;
+}